diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index cca57d2..f917130 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1861,6 +1861,9 @@ HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none", new StringSet("none", "idonly", "traverse", "execution"), ""), HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES("hive.explain.dependency.append.tasktype", false, ""), + HIVE_EXPLAIN_TRANSLATE_COLUMN_NAME("hive.explain.translate.column.name", false, + "Whether to translate internal column names to external ones for explain."), + HIVECOUNTERGROUP("hive.counters.group.name", "HIVE", "The name of counter group for internal Hive variables (CREATED_FILE, FATAL_ERROR, etc.)"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index 8e1ba48..09e2d04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -40,6 +40,7 @@ import java.util.TreeMap; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.DriverContext; @@ -554,6 +555,10 @@ private JSONObject outputPlan(Serializable work, PrintStream out, Operator operator = (Operator) work; if (operator.getConf() != null) { + OperatorDesc opDesc = operator.getConf(); + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_EXPLAIN_TRANSLATE_COLUMN_NAME)) { + opDesc.prepareExplainColumns(this.getWork().getParseContext().getExplainColCtx(), operator); + } String appender = isLogical ? " (" + operator.getOperatorId() + ")" : ""; JSONObject jsonOut = outputPlan(operator.getConf(), out, extended, jsonOutput, jsonOutput ? 0 : indent, appender); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java index c9e8086..14723d9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java @@ -529,6 +529,9 @@ protected MapJoinOperator convertJoinToBucketMapJoin( joinContext.getBigTablePosition(), false, false); + if (parseContext.getExplainColCtx() != null) { + parseContext.getExplainColCtx().getJoinOpLineage().put(mapJoinOp, joinOp); + } // Remove the join operator from the query join context parseContext.getMapJoinContext().put(mapJoinOp, parseContext.getJoinContext().get(joinOp)); parseContext.getJoinContext().remove(joinOp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index 04bafda..335a368 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -254,6 +254,9 @@ private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext cont .getSortCols()); mergeJoinOp.setOpTraits(opTraits); mergeJoinOp.setStatistics(joinOp.getStatistics()); + if (context.parseContext.getExplainColCtx() != null) { + context.parseContext.getExplainColCtx().getJoinOpLineage().put(mergeJoinOp, joinOp); + } for (Operator parentOp : joinOp.getParentOperators()) { int pos = parentOp.getChildOperators().indexOf(joinOp); @@ -606,6 +609,9 @@ public MapJoinOperator convertJoinMapJoin(JoinOperator joinOp, OptimizeTezProcCo MapJoinOperator mapJoinOp = MapJoinProcessor.convertJoinOpMapJoinOp(context.conf, parseContext.getOpParseCtx(), joinOp, parseContext.getJoinContext().get(joinOp), bigTablePosition, true); + if (context.parseContext.getExplainColCtx() != null) { + context.parseContext.getExplainColCtx().getJoinOpLineage().put(mapJoinOp, joinOp); + } Operator parentBigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 2d82cd8..85fc099 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -226,7 +226,7 @@ private static void genMapJoinLocalWork(MapredWork newWork, MapJoinOperator mapJ * @param mapJoinPos * @throws SemanticException */ - public static void genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, + public static MapJoinOperator genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, JoinOperator op, int mapJoinPos) throws SemanticException { LinkedHashMap, OpParseContext> opParseCtxMap = @@ -236,6 +236,7 @@ public static void genMapJoinOpAndLocalWork(HiveConf conf, MapredWork newWork, MapJoinOperator newMapJoinOp = MapJoinProcessor.convertMapJoin(conf, opParseCtxMap, op, newJoinTree, mapJoinPos, true, false); genLocalWorkForMapJoin(newWork, newMapJoinOp, mapJoinPos); + return newMapJoinOp; } public static void genLocalWorkForMapJoin(MapredWork newWork, MapJoinOperator newMapJoinOp, @@ -499,6 +500,9 @@ public MapJoinOperator generateMapJoinOperator(ParseContext pctx, JoinOperator o .getOpParseCtx(); MapJoinOperator mapJoinOp = convertMapJoin(pctx.getConf(), opParseCtxMap, op, joinTree, mapJoinPos, noCheckOuterJoin, true); + if (pctx.getExplainColCtx() != null) { + pctx.getExplainColCtx().getJoinOpLineage().put(mapJoinOp, op); + } // create a dummy select to select all columns genSelectPlan(pctx, mapJoinOp); return mapJoinOp; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java index 3df1c26..b3167aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator; +import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.TableScanOperator; @@ -178,8 +179,13 @@ private MapRedTask convertTaskToMapJoinTask(MapredWork newWork, int bigTablePosi JoinOperator newJoinOp = getJoinOp(newTask); // optimize this newWork given the big table position - MapJoinProcessor.genMapJoinOpAndLocalWork(physicalContext.getParseContext().getConf(), + MapJoinOperator mapJoinOp = MapJoinProcessor.genMapJoinOpAndLocalWork(physicalContext.getParseContext().getConf(), newWork, newJoinOp, bigTablePosition); + if (physicalContext.getParseContext().getExplainColCtx() != null) { + physicalContext.getParseContext().getExplainColCtx().getJoinOpLineage() + .put(mapJoinOp, newJoinOp); + } + return newTask; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java index 9076d48..4c4e14d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java @@ -228,6 +228,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, Object.. dummyOp.setChildOperators(dummyChildren); // add this dummy op to the dummp operator list dummyOperators.add(dummyOp); + if (context.getParseCtx().getExplainColCtx() != null) { + context.getParseCtx().getExplainColCtx().getJoinOpLineage() + .put(hashTableSinkOp, mapJoinOp); + } } hashTableSinkOp.setParentOperators(smallTablesParentOp); for (Operator op : dummyOperators) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 2c02bd4..0b65e1c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -1256,6 +1256,9 @@ private void fixupParentChildOperators(Operator op, case EXTRACT: case EVENT: vectorOp = OperatorFactory.getVectorOperator(op.getConf(), vContext); + if (HiveConf.getBoolVar(this.physicalContext.conf, HiveConf.ConfVars.HIVE_EXPLAIN_TRANSLATE_COLUMN_NAME)) { + this.physicalContext.getParseContext().getExplainColCtx().getVectorOpLineage().put(vectorOp, op); + } break; default: vectorOp = op; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainColumnContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainColumnContext.java new file mode 100644 index 0000000..2109938 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainColumnContext.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; + +/** + * ExplainColumnContext: The explain column context. This is used to track the column names + * for each operator after logical optimization and physical optimization. Later, the information + * is used for explain task to translate internal column names to external column names + * The basic idea to transform the internal name to external name is to + * (1) use snapshotLogicalPlanForExplain() to make a snapshot of a logical plan after logical optimization + * (2) for each operator in the explain task, call the ''prepareExplainColumn'' function for each operatorDesc + * (2.1) Each operator uses ''helpGetStartOp'' to map to a logical operator (start point) in the LogicalPlan + * (2.2) From start point, each operatorDesc uses ''findExternalName'' to track its external name + **/ + +public class ExplainColumnContext { + + private ParseContext pCtx; + private HashMap, List>> logicalPlan; + private HashMap, Operator> joinOpLineage; + private HashMap, Operator> vectorOpLineage; + private boolean vectorizeEnable; + + public ExplainColumnContext() { + } + + public HashMap, List>> getLogicalPlan() { + return logicalPlan; + } + + public void setLogicalPlan( + HashMap, List>> logicalPlan) { + this.logicalPlan = logicalPlan; + } + + public HashMap, Operator> getJoinOpLineage() { + return joinOpLineage; + } + + public void setJoinOpLineage( + HashMap, Operator> joinOpLineage) { + this.joinOpLineage = joinOpLineage; + } + + public HashMap, Operator> getVectorOpLineage() { + return vectorOpLineage; + } + + public void setVectorOpLineage( + HashMap, Operator> vectorOpLineage) { + this.vectorOpLineage = vectorOpLineage; + } + + // for non-join operator, byte is null. + // for join operator, byte is not null and it is used to pick the right parent. + public Operator helpGetStartOp(Operator operator, + Byte b) { + if (vectorizeEnable && this.vectorOpLineage.containsKey(operator)) { + operator = this.vectorOpLineage.get(operator); + } + String operatorId = operator.getOperatorId(); + for (Operator key : logicalPlan.keySet()) { + if (key.getOperatorId().equals(operatorId)) { + if (b == null) { + return key; + } else { + if (logicalPlan.get(key).get(b) != null) { + return logicalPlan.get(key).get(b); + } + } + } + } + // This is useful when a MAPJOIN op is created by a JOIN op + // or a serialized MAPJOIN op has operatorID the same as another MAPJOIN op, etc + for (Operator key : joinOpLineage.keySet()) { + if (key.getOperatorId().equals(operatorId)) { + Operator ret = helpGetStartOp(joinOpLineage.get(key), b); + if (ret != null) + return ret; + } + } + return null; + } + + public String[] findExternalName(String colName, Operator op) { + if (op == null) { + return null; + } + String[] result = null; + // first find it from its own RR. then find it from parents' RR + OpParseContext opCtx = pCtx.getOpParseCtx().get(op); + if (opCtx != null) { + RowResolver rr = opCtx.getRowResolver(); + if (rr != null) { + HashMap invRslvMap = rr.getInvRslvMap(); + if (invRslvMap.containsKey(colName)) { + return invRslvMap.get(colName); + } + } + } + // start from itself, search for a single parent, will terminate when it meets + // a JOIN or reaches the end + // this case is true when internal name already equals external name + if (logicalPlan.containsKey(op)) { + if (logicalPlan.get(op) != null && logicalPlan.get(op).size() == 1) { + result = findExternalName(colName, logicalPlan.get(op).get(0)); + } + } + return result; + } + + public void init(ParseContext pCtx) { + this.pCtx = pCtx; + logicalPlan = new HashMap, List>>(); + joinOpLineage = new HashMap, Operator>(); + vectorizeEnable = pCtx.getConf().getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED); + if (vectorizeEnable) { + vectorOpLineage = new HashMap, Operator>(); + } + List> ops = new ArrayList>(); + ops.addAll(pCtx.getTopOps().values()); + while (ops.size() > 0) { + Operator op = ops.get(0); + ops.remove(0); + List> parents = new ArrayList>(); + parents.addAll(op.getParentOperators()); + logicalPlan.put(op, parents); + ops.addAll(op.getChildOperators()); + } + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 8215c26..377ec36 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -113,6 +113,8 @@ private TableDesc fetchTabledesc; private Operator fetchSource; private ListSinkOperator fetchSink; + + private ExplainColumnContext explainColCtx; public ParseContext() { } @@ -182,6 +184,7 @@ public ParseContext( Map> opToPartToSkewedPruner, Map viewAliasToInput, List reduceSinkOperatorsAddedByEnforceBucketingSorting, + ExplainColumnContext explainColCtx, QueryProperties queryProperties) { this.conf = conf; this.qb = qb; @@ -214,6 +217,7 @@ public ParseContext( this.viewAliasToInput = viewAliasToInput; this.reduceSinkOperatorsAddedByEnforceBucketingSorting = reduceSinkOperatorsAddedByEnforceBucketingSorting; + this.explainColCtx = explainColCtx; this.queryProperties = queryProperties; } @@ -673,4 +677,16 @@ public ListSinkOperator getFetchSink() { public void setFetchSink(ListSinkOperator fetchSink) { this.fetchSink = fetchSink; } + + public ExplainColumnContext getExplainColCtx() { + return explainColCtx; + } + + public void setExplainColCtx(ExplainColumnContext explainColCtx) { + this.explainColCtx = explainColCtx; + } + + public void initializeAndSnapshotLogicalPlanForExplain() { + explainColCtx.init(this); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 88fd0fc..d6b51b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -341,6 +341,7 @@ List> listMapJoinOpsNoReducer; private HashMap opToSamplePruner; private final Map> opToPartToSkewedPruner; + private ExplainColumnContext explainColCtx; /** * a map for the split sampling, from alias to an instance of SplitSample * that describes percentage and number. @@ -429,6 +430,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { aliasToCTEs = new HashMap(); globalLimitCtx = new GlobalLimitCtx(); viewAliasToInput = new HashMap(); + explainColCtx = new ExplainColumnContext(); noscan = partialscan = false; } @@ -506,7 +508,7 @@ public ParseContext getParseContext() { listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, + reduceSinkOperatorsAddedByEnforceBucketingSorting, explainColCtx, queryProperties); } @@ -10105,7 +10107,7 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); + reduceSinkOperatorsAddedByEnforceBucketingSorting, explainColCtx, queryProperties); if (createVwDesc != null) { saveViewDefinition(); @@ -10150,6 +10152,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { LOG.debug("After logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); } + // snapshot the parent relationships of all the operators for + // explain before any other optimization. + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_EXPLAIN_TRANSLATE_COLUMN_NAME)) { + pCtx.initializeAndSnapshotLogicalPlanForExplain(); + } + // Generate column access stats if required - wait until column pruning takes place // during optimization boolean isColumnInfoNeedForAuth = SessionState.get().isAuthorizationModeV2() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 23fbbe1..6548564 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -396,6 +396,7 @@ public ParseContext getParseContext(ParseContext pCtx, List operator){ + }; + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java index 8b25c2b..8b61352 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java @@ -20,9 +20,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.Properties; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; /** * FileSinkDesc. @@ -421,4 +424,35 @@ public void setTransactionId(long id) { public long getTransactionId() { return txnId; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + if (operator.getParentOperators() != null && operator.getParentOperators().size() == 1) { + operator = explainColCtx.helpGetStartOp(operator.getParentOperators().get(0), null); + } else { + return; + } + Properties properties = tableInfo.getProperties(); + String cols = (String) properties.get("columns"); + String[] columnNames = cols.split(","); + String newcols = ""; + for (int index = 0; index < columnNames.length; index++) { + String[] results = explainColCtx.findExternalName(columnNames[index], operator); + if (index == 0) { + if (results != null) { + newcols += results[1]; + } else { + newcols += columnNames[index]; + } + } else { + if (results != null) { + newcols += "," + results[1]; + } else { + newcols += "," + columnNames[index]; + } + } + } + properties.setProperty("columns", newcols); + tableInfo.setProperties(properties); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index 5856743..6b48b98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; + /** * FilterDesc. @@ -147,4 +151,32 @@ public Object clone() { filterDesc.setSortedFilter(isSortedFilter()); return filterDesc; } + + // get all the ExprNodeColumnDesc + private List getColumnExpr(ExprNodeDesc expr) { + List ret = new ArrayList(); + if (expr.getChildren() != null) { + for (ExprNodeDesc children : expr.getChildren()) { + if (children instanceof ExprNodeColumnDesc) { + ret.add((ExprNodeColumnDesc) children); + } else { + ret.addAll(getColumnExpr(children)); + } + } + } + return ret; + } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + operator = explainColCtx.helpGetStartOp(operator, null); + for (ExprNodeColumnDesc colexpr : getColumnExpr(predicate)) { + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 7a0b0da..991a6bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hive.common.util.AnnotationUtils; @@ -296,5 +298,41 @@ public boolean isDistinct() { public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + operator = explainColCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + for (AggregationDesc agg : aggregators) { + ArrayList parameters = agg.getParameters(); + for (ExprNodeDesc expr : parameters) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + for (ExprNodeDesc expr : keys) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java index 1e0eb6b..dc85ee2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/HashTableSinkDesc.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -29,6 +30,9 @@ import java.util.Set; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; + /** * Map Join operator Descriptor implementation. @@ -397,4 +401,50 @@ public BucketMapJoinContext getBucketMapjoinContext() { public void setBucketMapjoinContext(BucketMapJoinContext bucketMapjoinContext) { this.bucketMapjoinContext = bucketMapjoinContext; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + Operator joinOp = explainColCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = explainColCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + if (getExprs() != null) { + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + if (getKeys() != null) { + for (Entry> entry : getKeys().entrySet()) { + Set set = needReverse.get(entry.getKey()); + if (set == null) + set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = explainColCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java index 0e2c6ee..ea7ff71 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java @@ -22,11 +22,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; /** @@ -540,4 +545,38 @@ public boolean isFixedAsSorted() { public void setFixedAsSorted(boolean fixedAsSorted) { this.fixedAsSorted = fixedAsSorted; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + Operator joinOp = explainColCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = explainColCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = explainColCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java index d43bd60..8551f78 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -28,6 +29,9 @@ import java.util.Map.Entry; import java.util.Set; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; + /** * Map Join operator Descriptor implementation. * @@ -345,4 +349,52 @@ public void setGenJoinKeys(boolean genJoinKeys) { public boolean getGenJoinKeys() { return genJoinKeys; } + + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + Operator joinOp = explainColCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = explainColCtx.findExternalName(colName, joinOp); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + HashMap> needReverse = new HashMap>(); + if (getExprs() != null) { + for (Entry> entry : getExprs().entrySet()) { + Set set = new HashSet(); + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + if (getKeys() != null) { + for (Entry> entry : getKeys().entrySet()) { + Set set = needReverse.get(entry.getKey()); + if (set == null) { + set = new HashSet(); + } + set.addAll(entry.getValue()); + needReverse.put(entry.getKey(), set); + } + } + for (Entry> entry : needReverse.entrySet()) { + Byte b = entry.getKey(); + Operator parentOp = explainColCtx.helpGetStartOp(operator, b); + Set colList = entry.getValue(); + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, parentOp); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index c8c9570..e45695c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -20,10 +20,16 @@ import java.io.Serializable; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; + public interface OperatorDesc extends Serializable, Cloneable { public Object clone() throws CloneNotSupportedException; public Statistics getStatistics(); public void setStatistics(Statistics statistics); public OpTraits getOpTraits(); public void setOpTraits(OpTraits opTraits); + public void prepareExplainColumns( + ExplainColumnContext explainColCtx, + Operator operator); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java index 57beb69..da82bce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java @@ -20,11 +20,15 @@ import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; /** @@ -407,4 +411,22 @@ public final void setReducerTraits(EnumSet traits) { public AcidUtils.Operation getWriteType() { return writeType; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + operator = explainColCtx.helpGetStartOp(operator, null); + Set needReverse = new HashSet(); + needReverse.addAll(keyCols); + needReverse.addAll(valueCols); + for (ExprNodeDesc expr : needReverse) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index fa6b548..05b776c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -21,6 +21,9 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.parse.ExplainColumnContext; + /** * SelectDesc. @@ -137,4 +140,27 @@ public boolean isSelStarNoCompute() { public void setSelStarNoCompute(boolean selStarNoCompute) { this.selStarNoCompute = selStarNoCompute; } + + @Override + public void prepareExplainColumns(ExplainColumnContext explainColCtx, Operator operator) { + operator = explainColCtx.helpGetStartOp(operator, null); + List outputColumnNames = this.getOutputColumnNames(); + for (int index = 0; index < outputColumnNames.size(); index++) { + String colName = outputColumnNames.get(index); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + outputColumnNames.set(index, results[1]); + } + } + for (ExprNodeDesc expr : colList) { + if (expr instanceof ExprNodeColumnDesc) { + ExprNodeColumnDesc colexpr = (ExprNodeColumnDesc) expr; + String colName = colexpr.getExprString(); + String[] results = explainColCtx.findExternalName(colName, operator); + if (results != null) { + colexpr.setColumn(results[1]); + } + } + } + } } diff --git a/ql/src/test/queries/clientpositive/explainColTest_1.q b/ql/src/test/queries/clientpositive/explainColTest_1.q new file mode 100644 index 0000000..471074d --- /dev/null +++ b/ql/src/test/queries/clientpositive/explainColTest_1.q @@ -0,0 +1,194 @@ +set hive.explain.translate.column.name=true; +set hive.cbo.enable=true; +set hive.auto.convert.join=false; + +--project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src; + +--self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key); + +--two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +--three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key); + +--groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key; + +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c; + +EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90; + +--order by +explain select key as k11, value as v from src1 where key < 80 order by v; + +--union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +--semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value; + +explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key; + + +--mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3; + + +explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2; + +explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc; + +explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key); + + +CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12); + +INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08'); + + +ANALYZE TABLE ss COMPUTE STATISTICS; +ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +ANALYZE TABLE sr COMPUTE STATISTICS; +ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +ANALYZE TABLE cs COMPUTE STATISTICS; +ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100; + +set hive.vectorized.execution.enabled=true; + +EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100; diff --git a/ql/src/test/queries/clientpositive/explainColTest_2.q b/ql/src/test/queries/clientpositive/explainColTest_2.q new file mode 100644 index 0000000..ef89149 --- /dev/null +++ b/ql/src/test/queries/clientpositive/explainColTest_2.q @@ -0,0 +1,194 @@ +set hive.explain.translate.column.name=true; +set hive.cbo.enable=true; +set hive.auto.convert.join=true; + +--project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src; + +--self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key); + +--two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE; + +EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +--three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key); + +--groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key; + +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c; + +EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90; + +--order by +explain select key as k11, value as v from src1 where key < 80 order by v; + +--union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key; + +--semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value; + +explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key; + +explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key; + + +--mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3; + + +explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2; + +explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc; + +explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key); + + +CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE; + +INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11); + +INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12); + +INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08'); + + +ANALYZE TABLE ss COMPUTE STATISTICS; +ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +ANALYZE TABLE sr COMPUTE STATISTICS; +ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +ANALYZE TABLE cs COMPUTE STATISTICS; +ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3; + +EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100; + +set hive.vectorized.execution.enabled=true; + +EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100; diff --git a/ql/src/test/results/clientpositive/explainColTest_1.q.out b/ql/src/test/results/clientpositive/explainColTest_1.q.out new file mode 100644 index 0000000..b204bba --- /dev/null +++ b/ql/src/test/results/clientpositive/explainColTest_1.q.out @@ -0,0 +1,3578 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: false + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_0:x, $hdt$_0:$hdt$_1:x] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-1 depends on stages: Stage-3 + Stage-0 depends on stages: Stage-1 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((11.0 = 11.0) and value is not null) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: false + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /src1 [$hdt$_0:$hdt$_1:$hdt$_2:x] + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:$hdt$_1:$hdt$_1:z] + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: false + TableScan + GatherStats: false + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) + auto parallelism: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: -mr-10001 + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + properties: + columns value,key + columns.types string,string + escape.delim \ + serialization.lib org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_0:y] +#### A masked pattern was here #### + Needs Tagging: true + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + outputColumnNames: value, value, key + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-3 is a root stage + Stage-2 depends on stages: Stage-3 + Stage-1 depends on stages: Stage-2 + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + TableScan + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string), value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string) + sort order: + + Map-reduce partition columns: k3 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string) + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + TableScan + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + outputColumnNames: value + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ss +POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ss +PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sr +POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sr +PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cs +POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cs +PREHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@ss +POSTHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@ss +POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@sr +POSTHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@sr +POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@cs +POSTHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@cs +POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k2 EXPRESSION [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +PREHOOK: Output: default@ss +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +POSTHOOK: Output: default@ss +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +PREHOOK: Output: default@sr +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +POSTHOOK: Output: default@sr +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +PREHOOK: Output: default@cs +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +POSTHOOK: Output: default@cs +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3, Stage-10 + Stage-5 depends on stages: Stage-4, Stage-8 + Stage-6 depends on stages: Stage-5 + Stage-7 depends on stages: Stage-6 + Stage-8 is a root stage + Stage-10 is a root stage + Stage-0 depends on stages: Stage-7 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + value expressions: v1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v1 (type: string) + sort order: + + Map-reduce partition columns: v1 (type: string) + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + TableScan + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-3 depends on stages: Stage-2 + Stage-4 depends on stages: Stage-3, Stage-10 + Stage-5 depends on stages: Stage-4, Stage-8 + Stage-6 depends on stages: Stage-5 + Stage-7 depends on stages: Stage-6 + Stage-8 is a root stage + Stage-10 is a root stage + Stage-0 depends on stages: Stage-7 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + value expressions: v1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v1 (type: string) + sort order: + + Map-reduce partition columns: v1 (type: string) + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + TableScan + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-8 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-10 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/explainColTest_2.q.out b/ql/src/test/results/clientpositive/explainColTest_2.q.out new file mode 100644 index 0000000..0ab2538 --- /dev/null +++ b/ql/src/test/results/clientpositive/explainColTest_2.q.out @@ -0,0 +1,4090 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:x + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + Position of Big Table: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [$hdt$_0:$hdt$_1:x] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-7 is a root stage + Stage-5 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-5 + Stage-2 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:y + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_1:$hdt$_2:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:y + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} {key} + keys: + 0 key (type: string) + 1 key (type: string) + Position of Big Table: 1 + $hdt$_0:$hdt$_1:$hdt$_2:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + Position of Big Table: 0 + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: ((11.0 = 11.0) and value is not null) (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, value, key + Position of Big Table: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Local Work: + Map Reduce Local Work + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:$hdt$_1:$hdt$_1:z] + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-2 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-8 is a root stage + Stage-5 depends on stages: Stage-8 + Stage-0 depends on stages: Stage-5 + +STAGE PLANS: + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + d + Fetch Operator + limit: -1 + x + Fetch Operator + limit: -1 + z + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + d + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + z + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 {key} + keys: + 0 value (type: string) + 1 value (type: string) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1, Stage-3, Stage-4 + Stage-3 is a root stage + Stage-4 is a root stage + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + TableScan + Union + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 3 Data size: 264 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Statistics: Num rows: 1 Data size: 88 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + c + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + c + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + c + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + c + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-6 is a root stage + Stage-2 depends on stages: Stage-6 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-6 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + b + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-4 is a root stage + Stage-3 depends on stages: Stage-4 + Stage-0 depends on stages: Stage-3 + +STAGE PLANS: + Stage: Stage-4 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_1:$hdt$_1:src1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_1:$hdt$_1:src1 + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {k1} {k2} {k3} + 1 + keys: + 0 k3 (type: string) + 1 k11 (type: string) + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + keys: + 0 k3 (type: string) + 1 k11 (type: string) + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-6 depends on stages: Stage-1, Stage-3 , consists of Stage-7, Stage-8, Stage-2 + Stage-7 has a backup stage: Stage-2 + Stage-4 depends on stages: Stage-7 + Stage-8 has a backup stage: Stage-2 + Stage-5 depends on stages: Stage-8 + Stage-2 + Stage-3 is a root stage + Stage-0 depends on stages: Stage-4, Stage-5, Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-6 + Conditional Operator + + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {k2} {k3} + 1 + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + $INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + TableScan + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-3 + Map Reduce + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-11 is a root stage + Stage-2 depends on stages: Stage-11 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-11 + Map Reduce Local Work + Alias -> Map Local Tables: + null-subquery1:$hdt$_0-subquery1-subquery1:$hdt$_0-subquery1:$hdt$_0:$hdt$_0:x1 + Fetch Operator + limit: -1 + null-subquery1:$hdt$_0-subquery1-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_1:x + Fetch Operator + limit: -1 + null-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_0:$hdt$_0:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + null-subquery1:$hdt$_0-subquery1-subquery1:$hdt$_0-subquery1:$hdt$_0:$hdt$_0:x1 + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + null-subquery1:$hdt$_0-subquery1-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_1:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + null-subquery2:$hdt$_0-subquery2:$hdt$_0:$hdt$_0:$hdt$_0:x + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Statistics: Num rows: 14 Data size: 108 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 564 Data size: 5950 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-4 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-4 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_1:x + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_1:x + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {key} + 1 + keys: + 0 key (type: string) + 1 key (type: string) + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 28 Data size: 297 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Union + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 194 Data size: 2060 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ss +POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ss +PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sr +POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sr +PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cs +POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cs +PREHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@ss +POSTHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@ss +POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@sr +POSTHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@sr +POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@cs +POSTHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@cs +POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k2 EXPRESSION [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +PREHOOK: Output: default@ss +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +POSTHOOK: Output: default@ss +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +PREHOOK: Output: default@sr +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +POSTHOOK: Output: default@sr +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +PREHOOK: Output: default@cs +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +POSTHOOK: Output: default@cs +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-30 is a root stage + Stage-21 depends on stages: Stage-30 + Stage-20 depends on stages: Stage-21, Stage-25 , consists of Stage-28, Stage-29, Stage-4 + Stage-28 has a backup stage: Stage-4 + Stage-18 depends on stages: Stage-28 + Stage-17 depends on stages: Stage-4, Stage-18, Stage-19, Stage-24 , consists of Stage-26, Stage-27, Stage-5 + Stage-26 has a backup stage: Stage-5 + Stage-15 depends on stages: Stage-26 + Stage-6 depends on stages: Stage-5, Stage-15, Stage-16 + Stage-7 depends on stages: Stage-6 + Stage-27 has a backup stage: Stage-5 + Stage-16 depends on stages: Stage-27 + Stage-5 + Stage-29 has a backup stage: Stage-4 + Stage-19 depends on stages: Stage-29 + Stage-4 + Stage-31 is a root stage + Stage-24 depends on stages: Stage-31 + Stage-32 is a root stage + Stage-25 depends on stages: Stage-32 + Stage-0 depends on stages: Stage-7 + +STAGE PLANS: + Stage: Stage-30 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_2:src1 + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_3:d1 + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_4:$hdt$_5:ss + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_2:src1 + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_3:d1 + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_4:$hdt$_5:ss + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + + Stage: Stage-21 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-20 + Conditional Operator + + Stage: Stage-28 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {v1} + 1 {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + + Stage: Stage-18 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-17 + Conditional Operator + + Stage: Stage-26 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + + Stage: Stage-15 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-27 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + + Stage: Stage-16 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + TableScan + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-29 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + + Stage: Stage-19 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-31 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_1:d1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_1:d1 + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + + Stage: Stage-24 + Map Reduce + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-32 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_4:$hdt$_4:sr + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_4:$hdt$_4:sr + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + + Stage: Stage-25 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-30 is a root stage + Stage-21 depends on stages: Stage-30 + Stage-20 depends on stages: Stage-21, Stage-25 , consists of Stage-28, Stage-29, Stage-4 + Stage-28 has a backup stage: Stage-4 + Stage-18 depends on stages: Stage-28 + Stage-17 depends on stages: Stage-4, Stage-18, Stage-19, Stage-24 , consists of Stage-26, Stage-27, Stage-5 + Stage-26 has a backup stage: Stage-5 + Stage-15 depends on stages: Stage-26 + Stage-6 depends on stages: Stage-5, Stage-15, Stage-16 + Stage-7 depends on stages: Stage-6 + Stage-27 has a backup stage: Stage-5 + Stage-16 depends on stages: Stage-27 + Stage-5 + Stage-29 has a backup stage: Stage-4 + Stage-19 depends on stages: Stage-29 + Stage-4 + Stage-31 is a root stage + Stage-24 depends on stages: Stage-31 + Stage-32 is a root stage + Stage-25 depends on stages: Stage-32 + Stage-0 depends on stages: Stage-7 + +STAGE PLANS: + Stage: Stage-30 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_2:src1 + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_3:d1 + Fetch Operator + limit: -1 + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_4:$hdt$_5:ss + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_2:src1 + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_3:d1 + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_3:$hdt$_4:$hdt$_5:ss + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + + Stage: Stage-21 + Map Reduce + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-20 + Conditional Operator + + Stage: Stage-28 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {v1} + 1 {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + + Stage: Stage-18 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-17 + Conditional Operator + + Stage: Stage-26 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + + Stage: Stage-15 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-6 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-7 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-27 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$INTNAME1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$INTNAME1 + TableScan + HashTable Sink Operator + condition expressions: + 0 {k3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + + Stage: Stage-16 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + TableScan + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-29 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$INTNAME + TableScan + HashTable Sink Operator + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + + Stage: Stage-19 + Map Reduce + Map Operator Tree: + TableScan + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-4 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + TableScan + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-31 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_1:d1 + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_1:d1 + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + + Stage: Stage-24 + Map Reduce + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-32 + Map Reduce Local Work + Alias -> Map Local Tables: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_4:$hdt$_4:sr + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + $hdt$_0:$hdt$_0:$hdt$_2:$hdt$_4:$hdt$_4:sr + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + + Stage: Stage-25 + Map Reduce + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + Local Work: + Map Reduce Local Work + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out b/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out new file mode 100644 index 0000000..98c77bf --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/explainColTest_1.q.out @@ -0,0 +1,3410 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [y] + Map 3 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [z] + Map 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 + Truncated Path -> Alias: + /src1 [x] + Reducer 2 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + outputColumnNames: value, value, key + Position of Big Table: 0 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Reducer 4 + Needs Tagging: false + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: value (type: string) + auto parallelism: true + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Map 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + outputColumnNames: key, value, key, value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string), value (type: string) + Reducer 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + outputColumnNames: key, value, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string), value (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string) + sort order: + + Map-reduce partition columns: k3 (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + outputColumnNames: k1, k2, k3, k11 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 5 <- Map 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + outputColumnNames: k1, k2, k3, k11, v + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string), v (type: string) + sort order: ++ + Map-reduce partition columns: k11 (type: string), v (type: string) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 10 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 5 <- Map 4 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 7 <- Map 6 (SIMPLE_EDGE), Map 8 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 10 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 8 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 9 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 5 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + outputColumnNames: value, key + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 7 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + outputColumnNames: value + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 5 <- Union 2 (CONTAINS) + Reducer 3 <- Map 4 (SIMPLE_EDGE), Union 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Map 4 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + outputColumnNames: key, key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 213 Data size: 2266 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ss +POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ss +PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sr +POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sr +PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cs +POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cs +PREHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@ss +POSTHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@ss +POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@sr +POSTHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@sr +POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@cs +POSTHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@cs +POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k2 EXPRESSION [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +PREHOOK: Output: default@ss +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +POSTHOOK: Output: default@ss +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +PREHOOK: Output: default@sr +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +POSTHOOK: Output: default@sr +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +PREHOOK: Output: default@cs +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +POSTHOOK: Output: default@cs +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 14 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) + Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) + Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + Map 11 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 12 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 14 + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Map 15 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 16 + Map Operator Tree: + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Map 17 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Map 9 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reducer 10 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reducer 13 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + value expressions: v1 (type: string), k2 (type: string), k3 (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v1 (type: string) + sort order: + + Map-reduce partition columns: v1 (type: string) + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + Reducer 5 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + Reducer 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 10 <- Map 14 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) + Reducer 13 <- Map 12 (SIMPLE_EDGE), Map 16 (SIMPLE_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 15 (SIMPLE_EDGE) + Reducer 3 <- Map 11 (SIMPLE_EDGE), Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Map 17 (SIMPLE_EDGE), Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 13 (SIMPLE_EDGE), Reducer 4 (SIMPLE_EDGE) + Reducer 6 <- Reducer 10 (SIMPLE_EDGE), Reducer 5 (SIMPLE_EDGE) + Reducer 7 <- Reducer 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + Map 11 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 12 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 14 + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Map 15 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 16 + Map Operator Tree: + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Map 17 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Map 9 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reducer 10 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + outputColumnNames: v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Reducer 13 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + value expressions: v1 (type: string), k2 (type: string), k3 (type: string) + Reducer 3 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v1 (type: string) + sort order: + + Map-reduce partition columns: v1 (type: string) + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Reducer 4 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string) + Reducer 5 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + outputColumnNames: k1, v1, k2, v2, v3 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v20 (type: string), v30 (type: string) + sort order: ++ + Map-reduce partition columns: v20 (type: string), v30 (type: string) + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k20 (type: string) + Reducer 6 + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + outputColumnNames: k3, v3, k1, v1, k20, v20 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Execution mode: vectorized + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + diff --git a/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out b/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out new file mode 100644 index 0000000..4c8d089 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/explainColTest_2.q.out @@ -0,0 +1,3331 @@ +PREHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +PREHOOK: type: QUERY +POSTHOOK: query: --project +explain +select key as k1, key as k2, key+1, 'constant' as k4 from src +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src + Select Operator + expressions: key (type: string), key (type: string), (UDFToDouble(key) + 1.0) (type: double), 'constant' (type: string) + outputColumnNames: k1, k2, c2, k4 + ListSink + +PREHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +PREHOOK: type: QUERY +POSTHOOK: query: --self join +EXPLAIN EXTENDED +SELECT x.key, y.value +FROM src x JOIN src y ON (x.key = y.key) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + Estimated key counts: Map 2 => 250 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 2 + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string) + outputColumnNames: x.key, y.value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns x.key,y.value + columns.types string:string + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + Map 2 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 1 + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [x] + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_j1 +POSTHOOK: query: --two joins +CREATE TABLE dest_j1(key STRING, value STRING, val2 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dest_j1 +PREHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +INSERT OVERWRITE TABLE dest_j1 +SELECT x.key, z.value, y.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_JOIN + TOK_JOIN + TOK_TABREF + TOK_TABNAME + src1 + x + TOK_TABREF + TOK_TABNAME + src + y + = + . + TOK_TABLE_OR_COL + x + key + . + TOK_TABLE_OR_COL + y + key + TOK_TABREF + TOK_TABNAME + srcpart + z + and + and + = + . + TOK_TABLE_OR_COL + x + value + . + TOK_TABLE_OR_COL + z + value + = + . + TOK_TABLE_OR_COL + z + ds + '2008-04-08' + = + . + TOK_TABLE_OR_COL + z + hr + 11 + TOK_INSERT + TOK_DESTINATION + TOK_TAB + TOK_TABNAME + dest_j1 + TOK_SELECT + TOK_SELEXPR + . + TOK_TABLE_OR_COL + x + key + TOK_SELEXPR + . + TOK_TABLE_OR_COL + z + value + TOK_SELEXPR + . + TOK_TABLE_OR_COL + y + value + + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + Stage-3 depends on stages: Stage-0 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + tag: 0 + value expressions: value (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src + numFiles 1 + numRows 500 + rawDataSize 5312 + serialization.ddl struct src { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src + name: default.src + Truncated Path -> Alias: + /src [y] + Map 2 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: value is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + Estimated key counts: Map 3 => 7 + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 3 + Position of Big Table: 0 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string), key (type: string) + outputColumnNames: value, key + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {value} {key} + Estimated key counts: Map 1 => 250 + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, value, key + input vertices: + 0 Map 1 + Position of Big Table: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), y.value (type: string), z.value (type: string) + outputColumnNames: x.key, z.value, y.value + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 1 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: hr=11 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + ds 2008-04-08 + hr 11 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + numFiles 1 + numRows 500 + partition_columns ds/hr + partition_columns.types string:string + rawDataSize 5312 + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 5812 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.srcpart + partition_columns ds/hr + partition_columns.types string:string + serialization.ddl struct srcpart { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.srcpart + name: default.srcpart + Truncated Path -> Alias: + /srcpart/ds=2008-04-08/hr=11 [z] + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Filter Operator + isSamplingPred: false + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + tag: 1 + value expressions: key (type: string) + auto parallelism: true + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: src1 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments defaultdefault + columns.types string:string +#### A masked pattern was here #### + name default.src1 + numFiles 1 + numRows 25 + rawDataSize 191 + serialization.ddl struct src1 { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.src1 + name: default.src1 + Truncated Path -> Alias: + /src1 [x] + + Stage: Stage-2 + Dependency Collection + + Stage: Stage-0 + Move Operator + tables: + replace: true +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value,val2 + columns.comments + columns.types string:string:string +#### A masked pattern was here #### + name default.dest_j1 + serialization.ddl struct dest_j1 { string key, string value, string val2} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.dest_j1 + + Stage: Stage-3 + Stats-Aggr Operator +#### A masked pattern was here #### + +PREHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +PREHOOK: type: QUERY +POSTHOOK: query: --three joins +EXPLAIN +SELECT x.key, z.value, y.value, d.key +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +JOIN dest_j1 d ON (d.val2 = z.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: d + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Filter Operator + predicate: val2 is not null (type: boolean) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Reduce Output Operator + key expressions: val2 (type: string) + sort order: + + Map-reduce partition columns: val2 (type: string) + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + value expressions: key (type: string) + Map 2 + Map Operator Tree: + TableScan + alias: z + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (value is not null and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: y + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {value} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, value, value + input vertices: + 0 Map 4 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: key, value, key, value + input vertices: + 1 Map 2 + Statistics: Num rows: 302 Data size: 3213 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} {value} + 1 {key} + keys: + 0 key (type: string) + 1 val2 (type: string) + outputColumnNames: key, value, value, key + input vertices: + 1 Map 1 + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string), value (type: string), key (type: string) + outputColumnNames: key, value, value, key + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 332 Data size: 3534 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 4 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (key is not null and value is not null) (type: boolean) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 7 Data size: 53 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: string) + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +PREHOOK: type: QUERY +POSTHOOK: query: --groupby +explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key order by c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: c (type: bigint) + sort order: + + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + value expressions: k (type: string) + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: k (type: string), c (type: bigint) + outputColumnNames: k, c + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT a.k, a.c +FROM (SELECT b.key as k, count(1) as c FROM src b GROUP BY b.key) a +WHERE a.k >= 90 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) >= 90.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: a.k (type: string), a.c (type: bigint) + outputColumnNames: a.k, a.c + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +PREHOOK: type: QUERY +POSTHOOK: query: --order by +explain select key as k11, value as v from src1 where key < 80 order by v +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) < 80.0) (type: boolean) + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 61 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +PREHOOK: type: QUERY +POSTHOOK: query: --union +explain + select unionsrc.key, count(1) FROM (select 'tst1' as key, count(1) as value from src s1 + UNION ALL + select 'tst2' as key, count(1) as value from src s2 + UNION ALL + select 'tst3' as key, count(1) as value from src s3) unionsrc group by unionsrc.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 4 <- Union 3 (SIMPLE_EDGE) + Reducer 6 <- Map 5 (SIMPLE_EDGE), Union 3 (CONTAINS) + Reducer 8 <- Map 7 (SIMPLE_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 5 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Map 7 + Map Operator Tree: + TableScan + alias: s1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: 1 (type: int) + outputColumnNames: $f0 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: hash + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) (type: bigint) + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst3' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 4 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: unionsrc.key (type: string), c1 (type: bigint) + outputColumnNames: unionsrc.key, c1 + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Reducer 6 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst2' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Reducer 8 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f0))) + mode: mergepartial + outputColumnNames: (tok_function count (. (tok_table_or_col $hdt$_0) $f0)) + Select Operator + expressions: 'tst1' (type: string) + outputColumnNames: key + Select Operator + expressions: $f0 (type: string), 1 (type: int) + outputColumnNames: $f0, $f1 + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f1))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string) + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f1)) (type: bigint) + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +PREHOOK: type: QUERY +POSTHOOK: query: --semijoin +explain select * from src a left semi join src1 b on a.key=b.key sort by a.key, a.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + outputColumnNames: key, value + input vertices: + 1 Map 1 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string), value (type: string) + sort order: ++ + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left outer join src b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + input vertices: + 1 Map 1 + 2 Map 2 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a right outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Right Outer Join0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 key (type: string) + 2 (. (tok_table_or_col c) key) (type: string) + outputColumnNames: key + input vertices: + 0 Map 4 + 2 Map 3 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from src a full outer join srcpart b on a.key = b.key left semi join src1 c on b.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col c) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col c) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col c) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Outer Join 0 to 1 + Left Semi Join 1 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key left outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 3 <- Map 1 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 3 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Left Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + input vertices: + 1 Map 1 + 2 Map 2 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src1 b on a.key = b.key right outer join src c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 1 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 3 <- Map 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Left Semi Join 0 to 1 + Right Outer Join0 to 2 + condition expressions: + 0 {key} + 1 + 2 + keys: + 0 key (type: string) + 1 (. (tok_table_or_col b) key) (type: string) + 2 key (type: string) + outputColumnNames: key + input vertices: + 0 Map 4 + 1 Map 1 + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +PREHOOK: type: QUERY +POSTHOOK: query: explain select a.key from srcpart a left semi join src b on a.key = b.key full outer join src1 c on a.key = c.key sort by a.key +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Group By Operator + keys: key (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col b) key) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col b) key) (type: string) + sort order: + + Map-reduce partition columns: (. (tok_table_or_col b) key) (type: string) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: c + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Map 5 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Reducer 2 + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Semi Join 0 to 1 + Outer Join 0 to 2 + condition expressions: + 0 {key} + 1 + 2 + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + Reducer 3 + Reduce Operator Tree: + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4400 Data size: 46745 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +PREHOOK: type: QUERY +POSTHOOK: query: --mix +explain +select * from +(select key as k11 from src1) hdt1 join +(select key as k1, key as k2, key as k3 from src) hdt0 +on hdt1.k11=hdt0.k3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} + keys: + 0 k3 (type: string) + 1 k11 (type: string) + outputColumnNames: k1, k2, k3, k11 + input vertices: + 1 Map 2 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k3 (type: string), hdt1.k11 (type: string), hdt0.k1 (type: string), hdt0.k2 (type: string) + outputColumnNames: hdt1.k11, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 2 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: k11 + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k11 (type: string) + sort order: + + Map-reduce partition columns: k11 (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from +(select key as k11, value as v from src1 where key < 80 order by v) hdt1 join +(select key as k1, key as k2, key as k3 from src limit 5) hdt0 +on hdt1.k11=hdt0.k3 and hdt1.v=hdt0.k2 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: src + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), key (type: string), key (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), k2 (type: string), k3 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(key) < 80.0) and value is not null) (type: boolean) + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v (type: string) + sort order: + + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + value expressions: k11 (type: string) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 5 + Statistics: Num rows: 5 Data size: 50 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(k3) < 80.0) and k2 is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, k2, k3 + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k3 (type: string), k2 (type: string) + sort order: ++ + Map-reduce partition columns: k3 (type: string), k2 (type: string) + Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string) + Reducer 4 + Reduce Operator Tree: + Select Operator + expressions: k11 (type: string), v (type: string) + outputColumnNames: k11, v + Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {k2} {k3} + 1 {k11} {v} + keys: + 0 k3 (type: string), k2 (type: string) + 1 k11 (type: string), v (type: string) + outputColumnNames: k1, k2, k3, k11, v + input vertices: + 0 Reducer 2 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: hdt0.k2 (type: string), hdt0.k3 (type: string), hdt1.k11 (type: string), hdt1.v (type: string), hdt0.k1 (type: string) + outputColumnNames: hdt1.k11, hdt1.v, hdt0.k1, hdt0.k2, hdt0.k3 + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 4 Data size: 33 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select unionsrc.k, unionsrc.v from ( +SELECT x1.key as k, y1.value as v +FROM src x1 JOIN src y1 ON (x1.key = y1.key) +UNION ALL +SELECT x.key as k, y.value as v +FROM src1 x JOIN src y ON (x.value = y.key) +UNION ALL +SELECT 'tst3' as k, y3.value as v +FROM src1 x3 JOIN src1 y3 ON (x3.value = y3.value) +) unionsrc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 6 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 4 <- Map 5 (BROADCAST_EDGE), Union 3 (CONTAINS) + Map 7 <- Map 1 (BROADCAST_EDGE), Union 3 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: x1 + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 value (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 6 + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 4 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: value is not null (type: boolean) + Select Operator + expressions: value (type: string) + outputColumnNames: value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {value} + keys: + 0 value (type: string) + 1 value (type: string) + outputColumnNames: value + input vertices: + 1 Map 5 + Select Operator + expressions: 'tst3' (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 5 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Map 6 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: value is not null (type: boolean) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 13 Data size: 99 Basic stats: COMPLETE Column stats: NONE + value expressions: key (type: string) + Map 7 + Map Operator Tree: + TableScan + alias: x1 + Filter Operator + predicate: key is not null (type: boolean) + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: key, value + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {value} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: value, key + input vertices: + 1 Map 1 + Select Operator + expressions: key (type: string), v (type: string) + outputColumnNames: k, v + Select Operator + expressions: unionsrc.k (type: string), unionsrc.v (type: string) + outputColumnNames: unionsrc.k, unionsrc.v + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 3 + Vertex: Union 3 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +PREHOOK: type: QUERY +POSTHOOK: query: explain +SELECT subq1.key, z.key +FROM +( SELECT x.key as key from src x where x.key < 10 + UNION ALL + SELECT x.key as key from src x +) subq1 +JOIN (SELECT x.key as key from src x where x.key > 200) z ON (z.key = subq1.key) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) + Map 4 <- Map 3 (BROADCAST_EDGE), Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (((UDFToDouble(key) < 10.0) and (UDFToDouble(key) > 200.0)) and key is not null) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + input vertices: + 1 Map 3 + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Map 3 + Map Operator Tree: + TableScan + alias: x + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: x + Filter Operator + predicate: (UDFToDouble(key) > 200.0) (type: boolean) + Select Operator + expressions: key (type: string) + outputColumnNames: key + Select Operator + expressions: key (type: string) + outputColumnNames: key + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} + 1 {key} + keys: + 0 key (type: string) + 1 key (type: string) + outputColumnNames: key, key + input vertices: + 1 Map 3 + Select Operator + expressions: subq1.key (type: string), z.key (type: string) + outputColumnNames: subq1.key, z.key + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ss +POSTHOOK: query: CREATE TABLE ss(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ss +PREHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@sr +POSTHOOK: query: CREATE TABLE sr(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@sr +PREHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cs +POSTHOOK: query: CREATE TABLE cs(k1 STRING,v1 STRING,k2 STRING,v2 STRING,k3 STRING,v3 STRING) STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cs +PREHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@ss +POSTHOOK: query: INSERT OVERWRITE TABLE ss +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=11) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@ss +POSTHOOK: Lineage: ss.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: ss.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@sr +POSTHOOK: query: INSERT OVERWRITE TABLE sr +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08' and z.hr=12) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@sr +POSTHOOK: Lineage: sr.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k2 SIMPLE [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.k3 EXPRESSION [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v2 SIMPLE [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: sr.v3 EXPRESSION [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@cs +POSTHOOK: query: INSERT OVERWRITE TABLE cs +SELECT x.key,x.value,y.key,y.value,z.key,z.value +FROM src1 x +JOIN src y ON (x.key = y.key) +JOIN srcpart z ON (x.value = z.value and z.ds='2008-04-08') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@cs +POSTHOOK: Lineage: cs.k1 SIMPLE [(src1)x.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k2 EXPRESSION [(src)y.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.k3 SIMPLE [(srcpart)z.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v1 SIMPLE [(src1)x.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v2 EXPRESSION [(src)y.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: cs.v3 SIMPLE [(srcpart)z.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +PREHOOK: Output: default@ss +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +POSTHOOK: Output: default@ss +PREHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@ss +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE ss COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ss +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +PREHOOK: Output: default@sr +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +POSTHOOK: Output: default@sr +PREHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@sr +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE sr COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@sr +#### A masked pattern was here #### +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +PREHOOK: Output: default@cs +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +POSTHOOK: Output: default@cs +PREHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +PREHOOK: type: QUERY +PREHOOK: Input: default@cs +#### A masked pattern was here #### +POSTHOOK: query: ANALYZE TABLE cs COMPUTE STATISTICS FOR COLUMNS k1,v1,k2,v2,k3,v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cs +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 5 (BROADCAST_EDGE) + Map 4 <- Map 9 (BROADCAST_EDGE) + Map 6 <- Map 1 (BROADCAST_EDGE), Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + Map 10 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: v2, k3, v3 + input vertices: + 0 Map 5 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: k2, v2, k3, v3 + input vertices: + 0 Map 9 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 1 Map 1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 0 Map 3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 0 Map 10 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + input vertices: + 1 Map 4 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + input vertices: + 0 Map 2 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Map 9 + Map Operator Tree: + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink + +PREHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +select ss.k1,sr.k2,cs.k3,count(ss.v1),count(sr.v2),count(cs.v3) +FROM +ss,sr,cs,src d1,src d2,src d3,src1,srcpart +where + ss.k1 = d1.key +and sr.k1 = d2.key +and cs.k1 = d3.key +and ss.k2 = sr.k2 +and ss.k3 = sr.k3 +and ss.v1 = src1.value +and ss.v2 = srcpart.value +and sr.v2 = cs.v2 +and sr.v3 = cs.v3 +and ss.v3='ssv3' +and sr.v1='srv1' +and src1.key = 'src1key' +and srcpart.key = 'srcpartkey' +and d1.value = 'd1value' +and d2.value in ('2000Q1','2000Q2','2000Q3') +and d3.value in ('2000Q1','2000Q2','2000Q3') +group by +ss.k1,sr.k2,cs.k3 +order by +ss.k1,sr.k2,cs.k3 +limit 100 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Map 2 <- Map 5 (BROADCAST_EDGE) + Map 4 <- Map 9 (BROADCAST_EDGE) + Map 6 <- Map 1 (BROADCAST_EDGE), Map 10 (BROADCAST_EDGE), Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE), Map 4 (BROADCAST_EDGE) + Reducer 7 <- Map 6 (SIMPLE_EDGE) + Reducer 8 <- Reducer 7 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ss + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v3 = 'ssv3') and v2 is not null) and k1 is not null) and v1 is not null) and k2 is not null) and k3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, k3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string) + sort order: + + Map-reduce partition columns: v2 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + Map 10 + Map Operator Tree: + TableScan + alias: src1 + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'src1key') and value is not null) (type: boolean) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: value (type: string) + sort order: + + Map-reduce partition columns: value (type: string) + Statistics: Num rows: 6 Data size: 45 Basic stats: COMPLETE Column stats: NONE + Map 2 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: v2, k3, v3 + input vertices: + 0 Map 5 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: v2 (type: string), v3 (type: string) + sort order: ++ + Map-reduce partition columns: v2 (type: string), v3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: k3 (type: string) + Map 3 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value = 'd1value') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: string) + sort order: + + Map-reduce partition columns: key (type: string) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map 4 + Map Operator Tree: + TableScan + alias: d1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((value) IN ('2000Q1', '2000Q2', '2000Q3') and key is not null) (type: boolean) + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: key + Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k2} {v2} {k3} {v3} + 1 + keys: + 0 k1 (type: string) + 1 key (type: string) + outputColumnNames: k2, v2, k3, v3 + input vertices: + 0 Map 9 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k2, v2, k3, v3 + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k2 (type: string), k3 (type: string) + sort order: ++ + Map-reduce partition columns: k2 (type: string), k3 (type: string) + Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), v3 (type: string) + Map 5 + Map Operator Tree: + TableScan + alias: cs + Statistics: Num rows: 170 Data size: 5890 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((k1 is not null and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, v2, k3, v3 + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 22 Data size: 762 Basic stats: COMPLETE Column stats: NONE + value expressions: v2 (type: string), k3 (type: string), v3 (type: string) + Map 6 + Map Operator Tree: + TableScan + alias: srcpart + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((key = 'srcpartkey') and value is not null) (type: boolean) + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: value (type: string) + outputColumnNames: value + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v2 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 1 Map 1 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 key (type: string) + 1 k1 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 0 Map 3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), v1 (type: string), k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, k3 + Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 + 1 {k1} {v1} {k2} {k3} + keys: + 0 value (type: string) + 1 v1 (type: string) + outputColumnNames: k1, v1, k2, k3 + input vertices: + 0 Map 10 + Statistics: Num rows: 665 Data size: 7069 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k1} {v1} + 1 {k2} {v2} {v3} + keys: + 0 k2 (type: string), k3 (type: string) + 1 k2 (type: string), k3 (type: string) + outputColumnNames: k1, v1, k2, v2, v3 + input vertices: + 1 Map 4 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k20 (type: string), v20 (type: string), v30 (type: string), k1 (type: string), v1 (type: string) + outputColumnNames: k20, v20, v30, k1, v1 + Statistics: Num rows: 731 Data size: 7775 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {k3} {v3} + 1 {k1} {v1} {k20} {v20} + keys: + 0 v2 (type: string), v3 (type: string) + 1 v20 (type: string), v30 (type: string) + outputColumnNames: k3, v3, k1, v1, k20, v20 + input vertices: + 0 Map 2 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k20 (type: string), $f2 (type: string), v1 (type: string), v20 (type: string), $f3 (type: string) + outputColumnNames: $f0, $f1, $f2, $f3, $f4, $f5 + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: hash + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + sort order: +++ + Map-reduce partition columns: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + Statistics: Num rows: 804 Data size: 8552 Basic stats: COMPLETE Column stats: NONE + value expressions: (tok_function count (. (tok_table_or_col $hdt$_0) $f3)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)) (type: bigint), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) (type: bigint) + Map 9 + Map Operator Tree: + TableScan + alias: sr + Statistics: Num rows: 85 Data size: 2945 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((((((v1 = 'srv1') and k1 is not null) and k2 is not null) and k3 is not null) and v2 is not null) and v3 is not null) (type: boolean) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: k1 (type: string), k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + outputColumnNames: k1, k2, v2, k3, v3 + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: k1 (type: string) + sort order: + + Map-reduce partition columns: k1 (type: string) + Statistics: Num rows: 2 Data size: 69 Basic stats: COMPLETE Column stats: NONE + value expressions: k2 (type: string), v2 (type: string), k3 (type: string), v3 (type: string) + Reducer 7 + Reduce Operator Tree: + Group By Operator + aggregations: count((tok_function count (. (tok_table_or_col $hdt$_0) $f3))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f4))), count((tok_function count (. (tok_table_or_col $hdt$_0) $f5))) + keys: (. (tok_table_or_col $hdt$_0) $f0) (type: string), (. (tok_table_or_col $hdt$_0) $f1) (type: string), (. (tok_table_or_col $hdt$_0) $f2) (type: string) + mode: mergepartial + outputColumnNames: (. (tok_table_or_col $hdt$_0) $f0), (. (tok_table_or_col $hdt$_0) $f1), (. (tok_table_or_col $hdt$_0) $f2), (tok_function count (. (tok_table_or_col $hdt$_0) $f3)), (tok_function count (. (tok_table_or_col $hdt$_0) $f4)), (tok_function count (. (tok_table_or_col $hdt$_0) $f5)) + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string) + sort order: +++ + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + value expressions: c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + Execution mode: vectorized + Reducer 8 + Reduce Operator Tree: + Select Operator + expressions: ss.k1 (type: string), sr.k2 (type: string), cs.k3 (type: string), c3 (type: bigint), c4 (type: bigint), c5 (type: bigint) + outputColumnNames: ss.k1, sr.k2, cs.k3, c3, c4, c5 + Statistics: Num rows: 402 Data size: 4276 Basic stats: COMPLETE Column stats: NONE + Limit + Number of rows: 100 + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 100 Data size: 1000 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: 100 + Processor Tree: + ListSink +