diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index e238ff1..1442c9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -40,6 +40,7 @@ import java.util.TreeMap; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.DriverContext; @@ -554,6 +555,10 @@ private JSONObject outputPlan(Serializable work, PrintStream out, Operator operator = (Operator) work; if (operator.getConf() != null) { + OperatorDesc opDesc = operator.getConf(); + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)) { + opDesc.prepareCBOExplain(this.getWork().getParseContext(), operator); + } String appender = isLogical ? " (" + operator.getOperatorId() + ")" : ""; JSONObject jsonOut = outputPlan(operator.getConf(), out, extended, jsonOutput, jsonOutput ? 0 : indent, appender); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java index c9e8086..7ae718e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java @@ -530,6 +530,7 @@ protected MapJoinOperator convertJoinToBucketMapJoin( false, false); // Remove the join operator from the query join context + parseContext.getJoinOpLineage().put(mapJoinOp, joinOp); parseContext.getMapJoinContext().put(mapJoinOp, parseContext.getJoinContext().get(joinOp)); parseContext.getJoinContext().remove(joinOp); convertMapJoinToBucketMapJoin(mapJoinOp, joinContext); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index bedc3ac..7b0dcf8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -251,6 +251,7 @@ private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext cont .getSortCols()); mergeJoinOp.setOpTraits(opTraits); mergeJoinOp.setStatistics(joinOp.getStatistics()); + context.parseContext.getJoinOpLineage().put(mergeJoinOp, joinOp); for (Operator parentOp : joinOp.getParentOperators()) { int pos = parentOp.getChildOperators().indexOf(joinOp); @@ -603,6 +604,7 @@ public MapJoinOperator convertJoinMapJoin(JoinOperator joinOp, OptimizeTezProcCo MapJoinOperator mapJoinOp = MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, parseContext.getOpParseCtx(), joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true); + context.parseContext.getJoinOpLineage().put(mapJoinOp, joinOp); Operator parentBigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 46dcfaf..56f550e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -226,7 +226,7 @@ private static void genMapJoinLocalWork(MapredWork newWork, MapJoinOperator mapJ * @param mapJoinPos * @throws SemanticException */ - public static void genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, + public static MapJoinOperator genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, JoinOperator op, int mapJoinPos) throws SemanticException { LinkedHashMap, OpParseContext> opParseCtxMap = @@ -236,6 +236,7 @@ public static void genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, MapJoinOperator newMapJoinOp = MapJoinProcessor.convertMapJoin(conf, opParseCtxMap, op, newJoinTree, mapJoinPos, true, false); genLocalWorkForMapJoin(newWork, newMapJoinOp, mapJoinPos); + return newMapJoinOp; } public static void genLocalWorkForMapJoin(MapredWork newWork, MapJoinOperator newMapJoinOp, @@ -499,6 +500,7 @@ public MapJoinOperator generateMapJoinOperator(ParseContext pctx, JoinOperator o .getOpParseCtx(); MapJoinOperator mapJoinOp = convertMapJoin(pctx.getConf(), opParseCtxMap, op, joinTree, mapJoinPos, noCheckOuterJoin, true); + pctx.getJoinOpLineage().put(mapJoinOp, op); // create a dummy select to select all columns genSelectPlan(pctx, mapJoinOp); return mapJoinOp; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index 1a4fcbf..b24094a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -178,8 +179,11 @@ private MapRedTask convertTaskToMapJoinTask(MapredWork newWork, int bigTablePosi JoinOperator newJoinOp = getJoinOp(newTask); // optimize this newWork given the big table position - MapJoinProcessor.genMapJoinOpAndLocalWork(physicalContext.getParseContext().getConf(), + MapJoinOperator mapJoinOp = MapJoinProcessor.genMapJoinOpAndLocalWork(physicalContext.getParseContext().getConf(), newWork, newJoinOp, bigTablePosition); + + physicalContext.getParseContext().getJoinOpLineage().put(mapJoinOp, newJoinOp); + return newTask; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java index 9076d48..26036f5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java @@ -228,6 +228,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. dummyOp.setChildOperators(dummyChildren); // add this dummy op to the dummp operator list dummyOperators.add(dummyOp); + context.getParseCtx().getJoinOpLineage().put(hashTableSinkOp, mapJoinOp); } hashTableSinkOp.setParentOperators(smallTablesParentOp); for (Operator op : dummyOperators) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 8215c26..cb39a11 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import java.io.Serializable; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -113,6 +114,10 @@ private TableDesc fetchTabledesc; private Operator fetchSource; private ListSinkOperator fetchSink; + + //use for explain task to translate internal colnames to external names + private HashMap, List>> logicalPlan; + private HashMap, Operator> joinOpLineage; public ParseContext() { } @@ -182,6 +187,8 @@ public ParseContext( Map> opToPartToSkewedPruner, Map viewAliasToInput, List reduceSinkOperatorsAddedByEnforceBucketingSorting, + HashMap, List>> logicalPlan, + HashMap, Operator> joinOpLineage, QueryProperties queryProperties) { this.conf = conf; this.qb = qb; @@ -214,6 +221,8 @@ public ParseContext( this.viewAliasToInput = viewAliasToInput; this.reduceSinkOperatorsAddedByEnforceBucketingSorting = reduceSinkOperatorsAddedByEnforceBucketingSorting; + this.logicalPlan = logicalPlan; + this.joinOpLineage = joinOpLineage; this.queryProperties = queryProperties; } @@ -673,4 +682,97 @@ public ListSinkOperator getFetchSink() { public void setFetchSink(ListSinkOperator fetchSink) { this.fetchSink = fetchSink; } + + public HashMap, List>> getLogicalPlan() { + return logicalPlan; + } + + public void setLogicalPlan( + HashMap, List>> logicalPlan) { + this.logicalPlan = logicalPlan; + } + + public HashMap, Operator> getJoinOpLineage() { + return joinOpLineage; + } + + public void setJoinOpLineage( + HashMap, Operator> joinOpLineage) { + this.joinOpLineage = joinOpLineage; + } + + + // The basic idea to transform the internal name to external name is to + // (1) use snapshotLogicalPlanForExplain() to make a snapshot of a logical plan after logical optimization + // (2) for each operator in the explain task, call the prepareCBOExplain function for each operatorDesc + // (2.1) Each operator uses ''helpGetStartOp'' to map to a logical operator (start point) in the LogicalPlan + // (2.2) From start point, each operatorDesc uses ''findExternalName'' to track its external name + + public void snapshotLogicalPlanForExplain() { + List> ops = new ArrayList>(); + ops.addAll(this.getTopOps().values()); + while (ops.size() > 0) { + Operator op = ops.get(0); + ops.remove(0); + List> parents = new ArrayList>(); + parents.addAll(op.getParentOperators()); + logicalPlan.put(op, parents); + ops.addAll(op.getChildOperators()); + } + } + + // for non-join operator, b is null. + // for join operator, b is not null and it is used to pick the right parent. + public Operator helpGetStartOp(Operator operator, + Byte b) { + String operatorId = operator.getOperatorId(); + for (Operator key : logicalPlan.keySet()) { + if (key.getOperatorId().equals(operatorId)) { + if (b == null) { + return key; + } else { + if (logicalPlan.get(key).get(b) != null) { + return logicalPlan.get(key).get(b); + } + } + } + } + // This is useful when MAPJOIN->JOIN + // or MAPJOIN[54] has operatorID MAPJOIN_53, etc + for (Operator key : joinOpLineage.keySet()) { + if (key.getOperatorId().equals(operatorId)) { + Operator ret = helpGetStartOp(joinOpLineage.get(key), b); + if (ret != null) + return ret; + } + } + return null; + } + + public String[] findExternalName(String colName, Operator op) { + if (op == null) { + return null; + } + String[] result = null; + // first find it from its own RR. then find it from parents' RR + OpParseContext opCtx = opParseCtx.get(op); + if (opCtx != null) { + RowResolver rr = opCtx.getRowResolver(); + if (rr != null) { + HashMap invRslvMap = rr.getInvRslvMap(); + if (invRslvMap.containsKey(colName)) { + return invRslvMap.get(colName); + } + } + } + // start from itself, search for a single parent, will terminate when it meets + // a JOIN or end + // this case is true when internal name already equals external name + if (logicalPlan.containsKey(op)) { + if (logicalPlan.get(op) != null && logicalPlan.get(op).size() == 1) { + result = findExternalName(colName, logicalPlan.get(op).get(0)); + } + } + return result; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 9c944b6..b23e197 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -342,6 +342,8 @@ List> listMapJoinOpsNoReducer; private HashMap opToSamplePruner; private final Map> opToPartToSkewedPruner; + private HashMap, List>> logicalPlan; + private HashMap, Operator> joinOpLineage; /** * a map for the split sampling, from alias to an instance of SplitSample * that describes percentage and number. @@ -425,6 +427,8 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL); autogenColAliasPrfxIncludeFuncName = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME); + logicalPlan = new HashMap, List>>(); + joinOpLineage = new HashMap, Operator>(); queryProperties = new QueryProperties(); opToPartToSkewedPruner = new HashMap>(); aliasToCTEs = new HashMap(); @@ -507,7 +511,7 @@ public ParseContext getParseContext() { listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, + reduceSinkOperatorsAddedByEnforceBucketingSorting, logicalPlan, joinOpLineage, queryProperties); } @@ -9962,7 +9966,7 @@ public void init(boolean clearPartsCache) { QB qb = new QB(null, null, false); this.qb = qb; } - + @Override @SuppressWarnings("nls") public void analyzeInternal(ASTNode ast) throws SemanticException { @@ -10118,7 +10122,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); + reduceSinkOperatorsAddedByEnforceBucketingSorting, logicalPlan, joinOpLineage, queryProperties); if (createVwDesc != null) { saveViewDefinition(); @@ -10162,7 +10166,11 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { if (LOG.isDebugEnabled()) { LOG.debug("After logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); } - + + // snapshot the parent relationships of all the operators for + // explain before any other optimization. + pCtx.snapshotLogicalPlanForExplain(); + // Generate column access stats if required - wait until column pruning takes place // during optimization boolean isColumnInfoNeedForAuth = SessionState.get().isAuthorizationModeV2() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 23fbbe1..47ae9a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -396,6 +396,8 @@ public ParseContext getParseContext(ParseContext pCtx, List operator){ + }; + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index 8b25c2b..5656f80 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -20,9 +20,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.Properties; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.parse.ParseContext; /** * FileSinkDesc. @@ -421,4 +424,35 @@ public void setTransactionId(long id) { public long getTransactionId() { return txnId; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + if (operator.getParentOperators() != null && operator.getParentOperators().size() == 1) { + operator = pCtx.helpGetStartOp(operator.getParentOperators().get(0), null); + } else { + return; + } + Properties properties = tableInfo.getProperties(); + String cols = (String) properties.get("columns"); + String[] columnNames = cols.split(","); + String newcols = ""; + for (int index = 0; index < columnNames.length; index++) { + String[] results = pCtx.findExternalName(columnNames[index], operator); + if (index == 0) { + if (results != null) { + newcols += results[1]; + } else { + newcols += columnNames[index]; + } + } else { + if (results != null) { + newcols += "," + results[1]; + } else { + newcols += "," + columnNames[index]; + } + } + } + properties.setProperty("columns", newcols); + tableInfo.setProperties(properties); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index 5856743..a9a337c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; + /** * FilterDesc. @@ -147,4 +151,32 @@ public Object clone() { filterDesc.setSortedFilter(isSortedFilter()); return filterDesc; } + + // get all the ExprNodeColumnDesc + private List getColumnExpr(ExprNodeDesc expr) { + List ret = new ArrayList(); + if (expr.getChildren() != null) { + for (ExprNodeDesc children : expr.getChildren()) { + if (children instanceof ExprNodeColumnDesc) { + ret.add((ExprNodeColumnDesc) children); + } else { + ret.addAll(getColumnExpr(children)); + } + } + } + return ret; + } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + operator = pCtx.helpGetStartOp(operator, null); + for (ExprNodeColumnDesc colexpr : getColumnExpr(predicate)) { + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 7a0b0da..a1255d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hive.common.util.AnnotationUtils; @@ -296,5 +298,41 @@ public boolean isDistinct() { public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + operator = pCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + for (AggregationDesc agg : aggregators) { + ArrayList parameters = agg.getParameters(); + for (ExprNodeDesc expr : parameters) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + for (ExprNodeDesc expr : keys) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 1e0eb6b..445108d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -29,6 +30,9 @@ import java.util.Set; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; + /** * Map Join operator Descriptor implementation. @@ -397,4 +401,50 @@ public BucketMapJoinContext getBucketMapjoinContext() { public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) { this.bucketMapjoinContext = bucketMapjoinContext; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + Operator joinOp = pCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = pCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + if (getExprs() != null) { + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + if (getKeys() != null) { + for (Entry> entry : getKeys().entrySet()) { + Set set = needReverse.get(entry.getKey()); + if (set == null) + set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = pCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index 0e2c6ee..0e5338a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -22,11 +22,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; /** @@ -540,4 +545,38 @@ public boolean isFixedAsSorted() { public void setFixedAsSorted(boolean fixedAsSorted) { this.fixedAsSorted = fixedAsSorted; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + Operator joinOp = pCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = pCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = pCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index d43bd60..02bc30a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -28,6 +29,9 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; + /** * Map Join operator Descriptor implementation. * @@ -345,4 +349,52 @@ public void setGenJoinKeys(boolean genJoinKeys) { public boolean getGenJoinKeys() { return genJoinKeys; } + + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + Operator joinOp = pCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = pCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + if (getExprs() != null) { + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + if (getKeys() != null) { + for (Entry> entry : getKeys().entrySet()) { + Set set = needReverse.get(entry.getKey()); + if (set == null) { + set = new HashSet(); + } + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = pCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index c8c9570..16830fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -20,10 +20,16 @@ import java.io.Serializable; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; + public interface OperatorDesc extends Serializable, Cloneable { public Object clone() throws CloneNotSupportedException; public Statistics getStatistics(); public void setStatistics(Statistics statistics); public OpTraits getOpTraits(); public void setOpTraits(OpTraits opTraits); + public void prepareCBOExplain( + ParseContext pCtx, + Operator operator); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 57beb69..c75726f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -20,11 +20,15 @@ import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.parse.ParseContext; /** @@ -407,4 +411,22 @@ public final void setReducerTraits(EnumSet traits) { public AcidUtils.Operation getWriteType() { return writeType; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + operator = pCtx.helpGetStartOp(operator, null); + Set needReverse = new HashSet(); + needReverse.addAll(keyCols); + needReverse.addAll(valueCols); + for (ExprNodeDesc expr : needReverse) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index fa6b548..7d18d69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ParseContext; /** @@ -137,4 +139,27 @@ public boolean isSelStarNoCompute() { public void setSelStarNoCompute(boolean selStarNoCompute) { this.selStarNoCompute = selStarNoCompute; } + + @Override + public void prepareCBOExplain(ParseContext pCtx, Operator operator) { + operator = pCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = pCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/test/queries/clientpositive/explainColTest_1.q b/ql/src/test/queries/clientpositive/explainColTest_1.q new file mode 100644 index 0000000..be5ca7a --- /dev/null +++ b/ql/src/test/queries/clientpositive/explainColTest_1.q @@ -0,0 +1,102 @@ +set hive.cbo.enable=true; +set hive.auto.convert.join=false; + +--project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src; + +--self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key); + +--two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +--three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key); + +--groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key; + +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c; + +EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90; + +--order by +explain select key as k11, value as v from src1 where key < 80 order by v; + +--union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +--semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value; + +explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key; + + +--mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3; + + +explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2; + +explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc; + +explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key); \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/explainColTest_2.q b/ql/src/test/queries/clientpositive/explainColTest_2.q new file mode 100644 index 0000000..e00f69c --- /dev/null +++ b/ql/src/test/queries/clientpositive/explainColTest_2.q @@ -0,0 +1,102 @@ +set hive.cbo.enable=true; +set hive.auto.convert.join=true; + +--project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src; + +--self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key); + +--two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +--three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key); + +--groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key; + +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c; + +EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90; + +--order by +explain select key as k11, value as v from src1 where key < 80 order by v; + +--union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +--semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value; + +explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key; + + +--mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3; + + +explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2; + +explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc; + +explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key); \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/explainColTest_1.q.out b/ql/src/test/results/clientpositive/explainColTest_1.q.out new file mode 100644 index 0000000..98de993 --- /dev/null +++ b/ql/src/test/results/clientpositive/explainColTest_1.q.out @@ -0,0 +1,2520 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: false + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_0:x, $hdt$_0:$hdt$_1:x] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((11.0 = 11.0) and value is not null) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: false + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /src1 [$hdt$_0:$hdt$_1:$hdt$_2:x] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:$hdt$_1:$hdt$_1:z] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: false + TableScan + GatherStats: false + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10001 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_0:y] +#### A masked pattern was here #### + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + outputColumnNames: value, value, key + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + TableScan + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string), value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string) + sort order: + + Map-reduce partition columns: k3 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string) + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + TableScan + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + outputColumnNames: value + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/explainColTest_2.q.out b/ql/src/test/results/clientpositive/explainColTest_2.q.out new file mode 100644 index 0000000..415c7b8 --- /dev/null +++ b/ql/src/test/results/clientpositive/explainColTest_2.q.out @@ -0,0 +1,2616 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:x + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + Position of Big Table: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_1:x] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:y + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_1:$hdt$_2:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:y + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} {key} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + $hdt$_0:$hdt$_1:$hdt$_2:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + Position of Big Table: 0 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((11.0 = 11.0) and value is not null) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, value, key + Position of Big Table: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:$hdt$_1:$hdt$_1:z] + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-8 is a root stage + Stage-5 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + z + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + c + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + c + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + c + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_1:$hdt$_1:src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_1:$hdt$_1:src1 + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {k1} {k2} {k3} + 1 + keys: + 0 k3 (type: string) + 1 k11 (type: string) + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + keys: + 0 k3 (type: string) + 1 k11 (type: string) + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1, Stage-3 , consists of Stage-7, Stage-8, Stage-2 + Stage-7 has a backup stage: Stage-2 + Stage-4 depends on stages: Stage-7 + Stage-8 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-8 + Stage-2 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-4, Stage-5, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {k2} {k3} + 1 + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + TableScan + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-11 is a root stage + Stage-2 depends on stages: Stage-11 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:$hdt$_0-subquery1-subquery1:$hdt$_0-subquery1:$hdt$_0:$hdt$_0:x1 + Fetch Operator + limit: -1 + null-subquery1:$hdt$_0-subquery1-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_1:x + Fetch Operator + limit: -1 + null-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_0:$hdt$_0:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:$hdt$_0-subquery1-subquery1:$hdt$_0-subquery1:$hdt$_0:$hdt$_0:x1 + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + null-subquery1:$hdt$_0-subquery1-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_1:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + null-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_0:$hdt$_0:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_1:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_1:x + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out b/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out new file mode 100644 index 0000000..1ee12e4 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out @@ -0,0 +1,2470 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [y] + Map 3 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [z] + Map 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 + Truncated Path -> Alias: + /src1 [x] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + outputColumnNames: value, value, key + Position of Big Table: 0 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Reducer 4 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) + auto parallelism: true + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string), value (type: string) + Reducer 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string) + sort order: + + Map-reduce partition columns: k3 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 10 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 8 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + outputColumnNames: value + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Map 4 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out b/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out new file mode 100644 index 0000000..12ffc5b --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out @@ -0,0 +1,2435 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + Estimated key counts: Map 2 => 250 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 2 + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Map 2 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [y] + Map 2 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + Estimated key counts: Map 3 => 7 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 3 + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + Estimated key counts: Map 1 => 250 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, value, key + input vertices: + 0 Map 1 + Position of Big Table: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [z] + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 + Truncated Path -> Alias: + /src1 [x] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + Map 2 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, value + input vertices: + 0 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: key, value, key, value + input vertices: + 1 Map 2 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + outputColumnNames: key, value, value, key + input vertices: + 1 Map 1 + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 4 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 100 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + outputColumnNames: key, value + input vertices: + 1 Map 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + input vertices: + 1 Map 1 + 2 Map 2 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + input vertices: + 0 Map 4 + 2 Map 3 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + input vertices: + 1 Map 1 + 2 Map 2 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + input vertices: + 0 Map 4 + 1 Map 1 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + keys: + 0 k3 (type: string) + 1 k11 (type: string) + outputColumnNames: k1, k2, k3, k11 + input vertices: + 1 Map 2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 2 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + input vertices: + 0 Reducer 2 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 7 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: x1 + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 6 + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 4 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: value is not null (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value + input vertices: + 1 Map 5 + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: x1 + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 1 + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + input vertices: + 1 Map 3 + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + input vertices: + 1 Map 3 + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink +