diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 33b67dd..5573458 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1274,6 +1274,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { // Statistics HIVESTATSAUTOGATHER("hive.stats.autogather", true, "A flag to gather statistics automatically during the INSERT OVERWRITE command."), + HIVESTATSCOLAUTOGATHER("hive.stats.column.autogather", false, + "A flag to gather column statistics automatically during the INSERT OVERWRITE command."), HIVESTATSDBCLASS("hive.stats.dbclass", "fs", new PatternSet("jdbc(:.*)", "hbase", "counter", "custom", "fs"), "The storage that stores temporary Hive statistics. In filesystem based statistics collection ('fs'), \n" + "each task writes statistics it has collected in a file on the filesystem, which will be aggregated \n" + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java index d688ef1..1f64c23 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java @@ -185,6 +185,11 @@ public void initialize(HiveConf hiveConf) { } } + public void initialize(Transform transform) { + transformations = new ArrayList(); + transformations.add(transform); + } + /** * Invoke all the transformations one-by-one, and alter the query plan. * diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 5cbc428..d87622c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -592,7 +592,7 @@ public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) { } } - private static void replaceASTChild(ASTNode child, ASTNode newChild) { + static void replaceASTChild(ASTNode child, ASTNode newChild) { ASTNode parent = (ASTNode) child.parent; int childIndex = child.childIndex; parent.deleteChild(childIndex); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java new file mode 100644 index 0000000..218df96 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -0,0 +1,293 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.parse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.Context; +import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.SelectOperator; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.optimizer.ConstantPropagate; +import org.apache.hadoop.hive.ql.optimizer.Optimizer; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; +import org.apache.hadoop.hive.ql.plan.LoadFileDesc; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.SelectDesc; + +/** + * ColumnStatsAutoGatherContext: This is passed to the compiler when set + * hive.stats.autogather=true during the INSERT OVERWRITE command. + * + **/ + +public class ColumnStatsAutoGatherContext { + + public static final String colStats = "colStats"; + public AnalyzeRewriteContext analyzeRewrite; + private final List loadFileWork = new ArrayList<>(); + private final SemanticAnalyzer sa; + private final HiveConf conf; + private final Operator op; + private final String tableName; + private final ASTNode tableAST; + private final List colNames = new ArrayList<>(); + private final List selectedPos = new ArrayList<>(); + private final List partitionColNames = new ArrayList<>(); + private boolean isPartitionedTable; + + public ColumnStatsAutoGatherContext( + SemanticAnalyzer sa, HiveConf conf, + Operator op, String tableName, ASTNode ast) throws SemanticException { + super(); + this.sa = sa; + this.conf = conf; + this.op = op; + this.tableName = tableName; + this.tableAST = ast; + Hive db; + try { + db = Hive.get(conf); + Table tbl = db.getTable(tableName); + for (FieldSchema fs : tbl.getCols()) { + colNames.add(fs.getName()); + } + for (FieldSchema fs : tbl.getPartCols()) { + partitionColNames.add(fs.getName()); + } + String selectedCols = tbl.getProperty(colStats); + if (selectedCols != null) { + String[] selected = selectedCols.split(","); + if (selected != null) { + for (String col : selected) { + int index = this.colNames.indexOf(col.trim()); + if (index == -1) { + throw new SemanticException("In insert overwrite table " + tableName + + " statement, can not find column " + col + " in colStats table properties from " + + Arrays.toString(colNames.toArray())); + } else { + selectedPos.add(index); + } + } + } + } + isPartitionedTable = (partitionColNames.size() != 0); + //else means that colStats was not yet configured. Assume all the columns are selected. + } catch (HiveException e) { + throw new SemanticException(e); + } + } + + public List getLoadFileWork() { + return loadFileWork; + } + + public AnalyzeRewriteContext getAnalyzeRewrite() { + return analyzeRewrite; + } + + public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) { + this.analyzeRewrite = analyzeRewrite; + } + + public void insertAnalyzePipeline() throws SemanticException{ + // 1. Generate the statement of analyze table [tablename] compute statistics for columns + String analyzeCommand = "analyze table " + tableName + + " compute statistics for columns " + genCols(); + + // 2. + // In non-partitioned table case, it will generate TS-SEL-GBY-RS-GBY-SEL-FS operator + // In static-partitioned table case, it will generate TS-FIL(partitionKey)-SEL-GBY(partitionKey)-RS-GBY-SEL-FS operator + // In dynamic-partitioned table case, it will generate TS-SEL-GBY(partitionKey)-RS-GBY-SEL-FS operator + Operator selOp = null; + try { + selOp = genSelOpForAnalyze(analyzeCommand); + } catch (IOException | ParseException e) { + throw new SemanticException(e); + } + + // 3. attach this SEL to the operator right before FS + op.getChildOperators().add(selOp); + selOp.getParentOperators().clear(); + selOp.getParentOperators().add(op); + + // 4. address the colExp, colList, etc for the SEL + replaceSelectOperatorProcess((SelectOperator)selOp, op); + } + + private String genCols() { + if (selectedPos.size() == 0) { + return ""; + } else { + StringBuilder builder = new StringBuilder(); + for (int pos : selectedPos) { + builder.append("," + colNames.get(pos)); + } + return builder.substring(1); + } + } + + @SuppressWarnings("rawtypes") + private Operator genSelOpForAnalyze(String analyzeCommand) throws IOException, ParseException, SemanticException{ + //0. initialization + Context ctx = new Context(conf); + ParseDriver pd = new ParseDriver(); + ASTNode tree = pd.parse(analyzeCommand, ctx); + tree = ParseUtils.findRootNonNullToken(tree); + + ASTNode oldTable = new CalcitePlanner.ASTSearcher().simpleBreadthFirstSearch(tree, + HiveParser.TOK_ANALYZE, HiveParser.TOK_TAB); + if (oldTable == null) { + throw new SemanticException("Cannot find insert overwrite table in " + tree.dump()); + } + ASTNode newDest = new CalcitePlanner.ASTSearcher().simpleBreadthFirstSearch(tableAST, + HiveParser.TOK_DESTINATION, HiveParser.TOK_TAB); + if (newDest == null) { + throw new SemanticException("Cannot find insert overwrite table in " + tableAST.dump()); + } + CalcitePlanner.replaceASTChild(oldTable, newDest); + + //1. get the ColumnStatsSemanticAnalyzer + BaseSemanticAnalyzer baseSem = SemanticAnalyzerFactory.get(conf, tree); + ColumnStatsSemanticAnalyzer colSem = (ColumnStatsSemanticAnalyzer) baseSem; + + //2. get the rewritten AST + ASTNode ast = colSem.getRewriteASTOnly(tree, this); + baseSem = SemanticAnalyzerFactory.get(conf, ast); + SemanticAnalyzer sem = (SemanticAnalyzer) baseSem; + QB qb = new QB(null, null, false); + ASTNode child = ast; + ParseContext subPCtx = ((SemanticAnalyzer) sem).getParseContext(); + subPCtx.setContext(ctx); + ((SemanticAnalyzer) sem).initParseCtx(subPCtx); + sem.doPhase1(child, qb, sem.initPhase1Ctx(), null); + sem.getMetaData(qb); + Operator operator = sem.genPlan(qb); + // If it contains static partition (SP, or SP mix DP), we need to do constant propagation. + if (isPartitionedTable) { + Optimizer optm = new Optimizer(); + optm.setPctx(subPCtx); + optm.initialize(new ConstantPropagate()); + subPCtx = optm.optimize(); + } + + //3. populate the load file work so that ColumnStatsTask can work + loadFileWork.addAll(sem.getLoadFileWork()); + + //4. because there is only one TS for analyze statement, we can get it. + operator = sem.topOps.values().iterator().next(); + + //5. get the first SEL after TS + while(!(operator instanceof SelectOperator)){ + operator = operator.getChildOperators().get(0); + } + return operator; + } + + /** + * @param operator : the select operator in the analyze statement + * @param input : the operator right before FS in the insert overwrite statement + * @throws SemanticException + */ + private void replaceSelectOperatorProcess(SelectOperator operator, Operator input) + throws SemanticException { + RowSchema selRS = operator.getSchema(); + ArrayList signature = new ArrayList<>(); + OpParseContext inputCtx = sa.opParseCtx.get(input); + RowResolver inputRR = inputCtx.getRowResolver(); + ArrayList columns = inputRR.getColumnInfos(); + ArrayList colList = new ArrayList(); + ArrayList columnNames = new ArrayList(); + Map columnExprMap = + new HashMap(); + // the column positions in the operator should be like this + // <----non-partition columns---->|<--static partition columns-->|<--dynamic partition columns--> + // ExprNodeColumnDesc | ExprNodeConstantDesc | ExprNodeColumnDesc + // from input | generate itself | from input + // | + + // there is no selected columns, select all non-partition columns + if (selectedPos.size() == 0) { + for (int i = 0; i < colNames.size(); i++) { + ColumnInfo col = columns.get(i); + colList.add(new ExprNodeColumnDesc(col)); + String internalName = selRS.getColumnNames().get(i); + columnNames.add(internalName); + columnExprMap.put(internalName, new ExprNodeColumnDesc(col)); + signature.add(selRS.getSignature().get(i)); + } + // there is selected columns + } else { + for (int i = 0; i < selectedPos.size(); i++) { + int index = selectedPos.get(i); + ColumnInfo col = columns.get(index); + colList.add(new ExprNodeColumnDesc(col)); + String internalName = selRS.getColumnNames().get(index); + columnNames.add(internalName); + columnExprMap.put(internalName, new ExprNodeColumnDesc(col)); + signature.add(selRS.getSignature().get(index)); + } + } + // if there is any partition column (in static partition or dynamic + // partition or mixed case) + for (int i = 0; i < partitionColNames.size(); i++) { + ExprNodeDesc exprNodeDesc = operator.getConf().getColList().get(colNames.size() + i); + // static partition columns + if (exprNodeDesc instanceof ExprNodeConstantDesc) { + colList.add(exprNodeDesc); + String internalName = selRS.getColumnNames().get(colNames.size() + i); + columnNames.add(internalName); + columnExprMap.put(internalName, operator.getColumnExprMap().get(internalName)); + signature.add(selRS.getSignature().get(colNames.size() + i)); + } + // dynamic partition columns + else { + int index = columns.size() - (partitionColNames.size() - i); + ColumnInfo col = columns.get(index); + colList.add(new ExprNodeColumnDesc(col)); + String internalName = selRS.getColumnNames().get(colNames.size() + i); + columnNames.add(internalName); + columnExprMap.put(internalName, new ExprNodeColumnDesc(col)); + signature.add(selRS.getSignature().get(colNames.size() + i)); + } + } + operator.setConf(new SelectDesc(colList, columnNames, true)); + operator.setColumnExprMap(columnExprMap); + selRS.setSignature(signature); + operator.setSchema(selRS); + } + + public String getTableName() { + return tableName; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index a5f0a7f..c877fab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.InvalidTableException; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde.serdeConstants; @@ -138,8 +137,13 @@ private void handlePartialPartitionSpec(Map partSpec) throws partValsSpecified += partSpec.get(partKey) == null ? 0 : 1; } try { - if ((partValsSpecified == tbl.getPartitionKeys().size()) && (db.getPartition(tbl, partSpec, false, null, false) == null)) { - throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() + " : " + partSpec); + // for static partition, it may not exist when HIVESTATSCOLAUTOGATHER is set to true + if (!conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER)) { + if ((partValsSpecified == tbl.getPartitionKeys().size()) + && (db.getPartition(tbl, partSpec, false, null, false) == null)) { + throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() + + " : " + partSpec); + } } } catch (HiveException he) { throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg() + " : " + partSpec); @@ -450,4 +454,45 @@ public void analyze(ASTNode ast, Context origCtx) throws SemanticException { analyzeInternal(originalTree); } } + + /** + * @param ast is the original analyze ast + * @param qb is the qb that calls this function + * @param sem is the semantic analyzer that calls this function + * @return + * @throws SemanticException + */ + public ASTNode getRewriteASTOnly(ASTNode ast, ColumnStatsAutoGatherContext context) throws SemanticException { + tbl = getTable(ast); + colNames = getColumnName(ast); + // Save away the original AST + originalTree = ast; + boolean isPartitionStats = isPartitionLevelStats(ast); + Map partSpec = null; + checkForPartitionColumns(colNames, + Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys())); + validateSpecifiedColumnNames(colNames); + if (conf.getBoolVar(ConfVars.HIVE_STATS_COLLECT_PART_LEVEL_STATS) && tbl.isPartitioned()) { + isPartitionStats = true; + } + + if (isPartitionStats) { + isTableLevel = false; + partSpec = getPartKeyValuePairsFromAST(tbl, ast, conf); + handlePartialPartitionSpec(partSpec); + } else { + isTableLevel = true; + } + colType = getColumnTypes(colNames); + int numBitVectors = getNumBitVectorsForNDVEstimation(conf); + rewrittenQuery = genRewrittenQuery(colNames, numBitVectors, partSpec, isPartitionStats); + rewrittenTree = genRewrittenTree(rewrittenQuery); + + context.analyzeRewrite = new AnalyzeRewriteContext(); + context.analyzeRewrite.setTableName(tbl.getDbName() + "." + tbl.getTableName()); + context.analyzeRewrite.setTblLvl(isTableLevel); + context.analyzeRewrite.setColName(colNames); + context.analyzeRewrite.setColType(colType); + return rewrittenTree; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 5872e8e..7a3600d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -76,6 +76,7 @@ private HashMap nameToSplitSample; private List loadTableWork; private List loadFileWork; + private List columnStatsAutoGatherContexts; private Context ctx; private HiveConf conf; private HashMap idToTableNameMap; @@ -154,6 +155,7 @@ public ParseContext( Set joinOps, Set smbMapJoinOps, List loadTableWork, List loadFileWork, + List columnStatsAutoGatherContexts, Context ctx, HashMap idToTableNameMap, int destTableId, UnionProcContext uCtx, List> listMapJoinOpsNoReducer, Map prunedPartitions, @@ -173,6 +175,7 @@ public ParseContext( this.smbMapJoinOps = smbMapJoinOps; this.loadFileWork = loadFileWork; this.loadTableWork = loadTableWork; + this.columnStatsAutoGatherContexts = columnStatsAutoGatherContexts; this.topOps = topOps; this.ctx = ctx; this.idToTableNameMap = idToTableNameMap; @@ -531,4 +534,13 @@ public void setCreateTable(CreateTableDesc createTableDesc) { this.createTableDesc = createTableDesc; } + public List getColumnStatsAutoGatherContexts() { + return columnStatsAutoGatherContexts; + } + + public void setColumnStatsAutoGatherContexts( + List columnStatsAutoGatherContexts) { + this.columnStatsAutoGatherContexts = columnStatsAutoGatherContexts; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java index 14a7e9c..506a5cc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java @@ -63,6 +63,7 @@ private final Set destGroupingSets; private final Map destToHaving; private final HashSet insertIntoTables; + private final Map insertOverwriteTables; private boolean isAnalyzeCommand; // used for the analyze command (statistics) private boolean isInsertToTable; // used for insert overwrite command (statistics) @@ -131,6 +132,7 @@ public QBParseInfo(String alias, boolean isSubQ) { destToOrderby = new HashMap(); destToLimit = new HashMap(); insertIntoTables = new HashSet(); + insertOverwriteTables = new HashMap(); destRollups = new HashSet(); destCubes = new HashSet(); destGroupingSets = new HashSet(); @@ -637,6 +639,10 @@ public boolean isPartialScanAnalyzeCommand() { public void setPartialScanAnalyzeCommand(boolean isPartialScanAnalyzeCommand) { this.isPartialScanAnalyzeCommand = isPartialScanAnalyzeCommand; } + + public Map getInsertOverwriteTables() { + return insertOverwriteTables; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index f05407d..9c1f635 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -240,6 +241,7 @@ protected LinkedHashMap, OpParseContext> opParseCtx; private List loadTableWork; private List loadFileWork; + private List columnStatsAutoGatherContexts; private final Map joinContext; private final Map smbMapJoinContext; private final HashMap topToTable; @@ -317,6 +319,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { topSelOps = new LinkedHashMap>(); loadTableWork = new ArrayList(); loadFileWork = new ArrayList(); + columnStatsAutoGatherContexts = new ArrayList(); opParseCtx = new LinkedHashMap, OpParseContext>(); joinContext = new HashMap(); smbMapJoinContext = new HashMap(); @@ -351,6 +354,7 @@ protected void reset(boolean clearPartsCache) { } loadTableWork.clear(); loadFileWork.clear(); + columnStatsAutoGatherContexts.clear(); topOps.clear(); topSelOps.clear(); destTableId = 1; @@ -409,7 +413,7 @@ public ParseContext getParseContext() { return new ParseContext(conf, opToPartPruner, opToPartList, topOps, new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), - loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, + loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, @@ -586,7 +590,7 @@ ASTNode getAST() { return this.ast; } - protected void setAST(ASTNode newAST) { + public void setAST(ASTNode newAST) { this.ast = newAST; } @@ -1168,11 +1172,18 @@ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plan ctx_1.nextNum++; boolean isTmpFileDest = false; if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) { - ASTNode ch = (ASTNode)ast.getChild(0); - if (ch.getToken().getType() == HiveParser.TOK_DIR - && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) { - ch = (ASTNode)ch.getChild(0); + ASTNode ch = (ASTNode) ast.getChild(0); + if (ch.getToken().getType() == HiveParser.TOK_DIR && ch.getChildCount() > 0 + && ch.getChild(0) instanceof ASTNode) { + ch = (ASTNode) ch.getChild(0); isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE; + } else { + if (ast.getToken().getType() == HiveParser.TOK_DESTINATION + && ast.getChild(0).getType() == HiveParser.TOK_TAB) { + String fullTableName = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), + SessionState.get().getCurrentDatabase()); + qbp.getInsertOverwriteTables().put(fullTableName, ast); + } } } @@ -8986,13 +8997,29 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, qb.getParseInfo().setOuterQueryLimit(limit.intValue()); } if (!SessionState.get().getHiveOperation().equals(HiveOperation.CREATEVIEW)) { - curr = genFileSinkPlan(dest, qb, curr); + Operator op = genFileSinkPlan(dest, qb, curr); + // the following code is used to collect column stats when + // hive.stats.autogather=true + // and it is an insert to table + if (op instanceof FileSinkOperator) { + FileSinkOperator fsOp = (FileSinkOperator) op; + String tableName = fsOp.getConf().getTableInfo().getTableName(); + if (tableName != null && conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) + && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) + && qb.getParseInfo().getInsertOverwriteTables().containsKey(tableName)) { + ColumnStatsAutoGatherContext columnStatsAutoGatherContext = new ColumnStatsAutoGatherContext( + this, conf, curr, tableName, qb.getParseInfo().getInsertOverwriteTables().get(tableName)); + columnStatsAutoGatherContext.insertAnalyzePipeline(); + columnStatsAutoGatherContexts.add(columnStatsAutoGatherContext); + } + } + curr = op; } } return curr; } - + @SuppressWarnings("nls") private Operator genUnionPlan(String unionalias, String leftalias, Operator leftOp, String rightalias, Operator rightOp) @@ -10138,7 +10165,7 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce ParseContext pCtx = new ParseContext(conf, opToPartPruner, opToPartList, topOps, new HashSet(joinContext.keySet()), new HashSet(smbMapJoinContext.keySet()), - loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, + loadTableWork, loadFileWork, columnStatsAutoGatherContexts, ctx, idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, @@ -12219,4 +12246,12 @@ private void warn(String msg) { SessionState.getConsole().printInfo( String.format("Warning: %s", msg)); } + + public List getLoadFileWork() { + return loadFileWork; + } + + public void setLoadFileWork(List loadFileWork) { + this.loadFileWork = loadFileWork; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index ba11e41..0f08ab5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -209,8 +209,22 @@ public void compile(final ParseContext pCtx, final List> tsks = new ArrayList<>(); + // find the move task that corresponds to the columnStatsAutoGatherContext + for (Task task : mvTask) { + if (task.getWork().getLoadTableWork().getTable().getTableName() + .equals(columnStatsAutoGatherContext.getTableName())) { + tsks.add(task); + break; + } + } + genColumnStatsTask(columnStatsAutoGatherContext.getAnalyzeRewrite(), + columnStatsAutoGatherContext.getLoadFileWork(), tsks, outerQueryLimit); + } } // For each task, set the key descriptor for the reducer @@ -307,7 +321,7 @@ public void compile(final ParseContext pCtx, final List loadTableWork, + protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, List loadFileWork, List> rootTasks, int outerQueryLimit) { ColumnStatsTask cStatsTask = null; ColumnStatsWork cStatsWork = null; @@ -394,7 +408,7 @@ public ParseContext getParseContext(ParseContext pCtx, List10; + +explain select * from nzhang_part14; + + +drop table nzhang_part14; +create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key,value'); + +INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10; + +describe extended nzhang_part14; + +explain select * from nzhang_part14; + + +drop table nzhang_part14; +create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value'); + +INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10; + +describe extended nzhang_part14; + +explain select * from nzhang_part14; + +explain select value from nzhang_part14; + + +drop table nzhang_part14; +create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key'); + +INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10; + +describe extended nzhang_part14; + +explain select * from nzhang_part14; + +explain select value from nzhang_part14; + +explain select key from nzhang_part14; + + +drop table a; +create table a (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key'); + +drop table b; +create table b (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value'); + +drop table c; +create table c (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value'); + + +FROM srcpart +INSERT OVERWRITE TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT OVERWRITE TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT OVERWRITE TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0; + +explain select * from a; +explain select key from a; +explain select value from b; +explain select key from b; +explain select value from c; +explain select key from c; + diff --git a/ql/src/test/results/clientpositive/autoColumnStats.q.out b/ql/src/test/results/clientpositive/autoColumnStats.q.out new file mode 100644 index 0000000..d2bdad1 --- /dev/null +++ b/ql/src/test/results/clientpositive/autoColumnStats.q.out @@ -0,0 +1,1362 @@ +PREHOOK: query: drop table src_multi1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi1 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: create table src_multi1 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi1 +PREHOOK: query: insert overwrite table src_multi1 select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@src_multi1 +POSTHOOK: query: insert overwrite table src_multi1 select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@src_multi1 +POSTHOOK: Lineage: src_multi1.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended select * from src_multi1 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select * from src_multi1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src_multi1 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_multi1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: describe extended src_multi1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi1 +POSTHOOK: query: describe extended src_multi1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi1 +key string default +value string default + +#### A masked pattern was here #### +PREHOOK: query: describe formatted src_multi1.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi1 +POSTHOOK: query: describe formatted src_multi1.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi1 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 205 2.812 3 from deserializer +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table a like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: from src +insert overwrite table a select * +insert overwrite table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert overwrite table a select * +insert overwrite table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 205 2.812 3 from deserializer +PREHOOK: query: describe formatted b.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 205 2.812 3 from deserializer +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@a +PREHOOK: Output: default@a +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@a +POSTHOOK: Output: default@a +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@b +PREHOOK: Output: default@b +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@b +POSTHOOK: Output: default@b +PREHOOK: query: create table a like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: create table b like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: from src +insert overwrite table a select * +insert into table b select * +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@a +PREHOOK: Output: default@b +POSTHOOK: query: from src +insert overwrite table a select * +insert into table b select * +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@a +POSTHOOK: Output: default@b +POSTHOOK: Lineage: a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted a.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@a +POSTHOOK: query: describe formatted a.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@a +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 205 2.812 3 from deserializer +PREHOOK: query: describe formatted b.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@b +POSTHOOK: query: describe formatted b.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@b +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string from deserializer +PREHOOK: query: drop table src_multi2 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_multi2 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_multi2 like src +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: create table src_multi2 like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_multi2 +PREHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_multi2 +POSTHOOK: query: insert overwrite table src_multi2 select subq.key, src.value from (select * from src union select * from src1)subq join src on subq.key=src.key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_multi2 +POSTHOOK: Lineage: src_multi2.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), (src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_multi2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain extended select * from src_multi2 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select * from src_multi2 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + src_multi2 + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + + +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: src_multi2 + Statistics: Num rows: 508 Data size: 5400 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 508 Data size: 90424 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: describe extended src_multi2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi2 +POSTHOOK: query: describe extended src_multi2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi2 +key string default +value string default + +#### A masked pattern was here #### +PREHOOK: query: describe formatted src_multi2.key +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_multi2 +POSTHOOK: query: describe formatted src_multi2.key +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_multi2 +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 205 2.8149606299212597 3 from deserializer +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string) + partitioned by (value string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: describe extended nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe extended nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +key string +value string + +# Partition Information +# col_name data_type comment + +value string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from ( + select * from (select 'k1' as key, cast(null as string) as value from src limit 2)a + union all + select * from (select 'k2' as key, '' as value from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@value= +POSTHOOK: Output: default@nzhang_part14@value=__HIVE_DEFAULT_PARTITION__ +POSTHOOK: Lineage: nzhang_part14 PARTITION(value= ).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=__HIVE_DEFAULT_PARTITION__).key EXPRESSION [] +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 6 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1620 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src5 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src5 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src5 as select key, value from src limit 5 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@src +PREHOOK: Output: database:default +PREHOOK: Output: default@src5 +POSTHOOK: query: create table src5 as select key, value from src limit 5 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@src +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src5 +PREHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from src5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src5 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(value) +select key, value from src5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src5 +POSTHOOK: Output: default@nzhang_part14@value=val_165 +POSTHOOK: Output: default@nzhang_part14@value=val_238 +POSTHOOK: Output: default@nzhang_part14@value=val_27 +POSTHOOK: Output: default@nzhang_part14@value=val_311 +POSTHOOK: Output: default@nzhang_part14@value=val_86 +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_165).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_238).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_27).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_311).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(value=val_86).key SIMPLE [(src5)src5.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 11 Data size: 25 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 11 Data size: 2970 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table alter5_src +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table alter5_src +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table alter5_src ( col1 string ) stored as textfile +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter5_src +POSTHOOK: query: create table alter5_src ( col1 string ) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter5_src +PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table alter5_src +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@alter5_src +POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite into table alter5_src +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@alter5_src +PREHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alter5 +POSTHOOK: query: create table alter5 ( col1 string ) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter5 +PREHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@alter5 +POSTHOOK: query: alter table alter5 add partition (dt='a') location 'parta' +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@alter5 +POSTHOOK: Output: default@alter5@dt=a +PREHOOK: query: describe extended alter5 partition (dt='a') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@alter5 +POSTHOOK: query: describe extended alter5 partition (dt='a') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@alter5 +col1 string +dt string + +# Partition Information +# col_name data_type comment + +dt string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table alter5 partition (dt='a') select col1 from alter5_src +PREHOOK: type: QUERY +PREHOOK: Input: default@alter5_src +PREHOOK: Output: default@alter5@dt=a +POSTHOOK: query: insert overwrite table alter5 partition (dt='a') select col1 from alter5_src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter5_src +POSTHOOK: Output: default@alter5@dt=a +POSTHOOK: Lineage: alter5 PARTITION(dt=a).col1 SIMPLE [(alter5_src)alter5_src.FieldSchema(name:col1, type:string, comment:null), ] +PREHOOK: query: explain select * from alter5 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from alter5 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: alter5 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col1 (type: string), dt (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1614 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select * from alter5 where dt='a' +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from alter5 where dt='a' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: alter5 + Statistics: Num rows: 6 Data size: 6 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col1 (type: string), 'a' (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 6 Data size: 1020 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table src_stat_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table src_stat_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_stat_part +POSTHOOK: query: create table src_stat_part(key string, value string) partitioned by (partitionId int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@src_stat_part +PREHOOK: query: insert overwrite table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: query: insert overwrite table src_stat_part partition (partitionId=1) +select * from src1 limit 5 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=1 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=1).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part.key PARTITION(partitionId=1) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part.key PARTITION(partitionId=1) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key string 0 3 1.2 3 from deserializer +PREHOOK: query: insert overwrite table src_stat_part partition (partitionId=2) +select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: query: insert overwrite table src_stat_part partition (partitionId=2) +select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@src_stat_part@partitionid=2 +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: src_stat_part PARTITION(partitionid=2).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe formatted src_stat_part.value PARTITION(partitionId=2) +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@src_stat_part +POSTHOOK: query: describe formatted src_stat_part.value PARTITION(partitionId=2) +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@src_stat_part +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +value string 0 14 4.92 7 from deserializer +PREHOOK: query: drop table srcbucket_mapjoin +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin(key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin +PREHOOK: query: drop table tab_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tab_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab_part +POSTHOOK: query: CREATE TABLE tab_part (key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab_part +PREHOOK: query: drop table srcbucket_mapjoin_part +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table srcbucket_mapjoin_part +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part (key int, value string) partitioned by (ds string) CLUSTERED BY (key) INTO 4 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@srcbucket_mapjoin_part +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket21.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket22.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: query: load data local inpath '../../data/files/srcbucket23.txt' INTO TABLE srcbucket_mapjoin_part partition(ds='2008-04-08') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin_part +PREHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +PREHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab_part partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin_part +POSTHOOK: Input: default@srcbucket_mapjoin_part@ds=2008-04-08 +POSTHOOK: Output: default@tab_part@ds=2008-04-08 +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab_part PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin_part)srcbucket_mapjoin_part.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab_part.key partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab_part +POSTHOOK: query: describe formatted tab_part.key partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab_part +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key int 0 498 0 196 from deserializer +PREHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tab +POSTHOOK: query: CREATE TABLE tab(key int, value string) PARTITIONED BY(ds STRING) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tab +PREHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +PREHOOK: type: QUERY +PREHOOK: Input: default@srcbucket_mapjoin +PREHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +PREHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: query: insert overwrite table tab partition (ds='2008-04-08') +select key,value from srcbucket_mapjoin +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcbucket_mapjoin +POSTHOOK: Input: default@srcbucket_mapjoin@ds=2008-04-08 +POSTHOOK: Output: default@tab@ds=2008-04-08 +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).key SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:key, type:int, comment:null), ] +POSTHOOK: Lineage: tab PARTITION(ds=2008-04-08).value SIMPLE [(srcbucket_mapjoin)srcbucket_mapjoin.FieldSchema(name:value, type:string, comment:null), ] +PREHOOK: query: describe formatted tab.key partition (ds='2008-04-08') +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@tab +POSTHOOK: query: describe formatted tab.key partition (ds='2008-04-08') +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@tab +# col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment + +key int 0 497 0 117 from deserializer +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) + partitioned by (ds string, hr string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: describe extended nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe extended nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +key string +value string +ds string +hr string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +#### A masked pattern was here #### +PREHOOK: query: insert overwrite table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: insert overwrite table nzhang_part14 partition(ds, hr) +select key, value, ds, hr from ( + select * from (select 'k1' as key, cast(null as string) as value, '1' as ds, '2' as hr from src limit 2)a + union all + select * from (select 'k2' as key, '' as value, '1' as ds, '3' as hr from src limit 2)b + union all + select * from (select 'k3' as key, ' ' as value, '2' as ds, '1' as hr from src limit 2)c +) T +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=2 +POSTHOOK: Output: default@nzhang_part14@ds=1/hr=3 +POSTHOOK: Output: default@nzhang_part14@ds=2/hr=1 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=2).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=1,hr=3).value EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).key EXPRESSION [] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2,hr=1).value EXPRESSION [] +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 6 Data size: 3064 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2006 Data size: 21272 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2006 Data size: 1081064 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key,value') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key,value') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe extended nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe extended nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +key string +value string +ds string +hr string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +#### A masked pattern was here #### +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe extended nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe extended nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +key string +value string +ds string +hr string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +#### A masked pattern was here #### +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: explain select value from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table nzhang_part14 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@nzhang_part14 +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: drop table nzhang_part14 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@nzhang_part14 +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@nzhang_part14 +POSTHOOK: query: create table if not exists nzhang_part14 (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@nzhang_part14 +PREHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@nzhang_part14@ds=2010-03-03 +POSTHOOK: query: INSERT OVERWRITE TABLE nzhang_part14 PARTITION (ds='2010-03-03', hr) +SELECT key, value, hr FROM srcpart WHERE ds is not null and hr>10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=11 +POSTHOOK: Output: default@nzhang_part14@ds=2010-03-03/hr=12 +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: nzhang_part14 PARTITION(ds=2010-03-03,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe extended nzhang_part14 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@nzhang_part14 +POSTHOOK: query: describe extended nzhang_part14 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@nzhang_part14 +key string +value string +ds string +hr string + +# Partition Information +# col_name data_type comment + +ds string +hr string + +#### A masked pattern was here #### +PREHOOK: query: explain select * from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 1092000 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: explain select value from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: explain select key from nzhang_part14 +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from nzhang_part14 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: nzhang_part14 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: drop table a +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@a +PREHOOK: Output: default@a +POSTHOOK: query: drop table a +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@a +POSTHOOK: Output: default@a +PREHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@a +POSTHOOK: query: create table a (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='key') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@a +PREHOOK: query: drop table b +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@b +PREHOOK: Output: default@b +POSTHOOK: query: drop table b +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@b +POSTHOOK: Output: default@b +PREHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@b +POSTHOOK: query: create table b (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@b +PREHOOK: query: drop table c +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table c +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@c +POSTHOOK: query: create table c (key string, value string) +partitioned by (ds string, hr string) tblproperties ('colStats'='value') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@c +PREHOOK: query: FROM srcpart +INSERT OVERWRITE TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT OVERWRITE TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT OVERWRITE TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@a@ds=2010-03-11 +PREHOOK: Output: default@b@ds=2010-04-11 +PREHOOK: Output: default@c@ds=2010-05-11 +POSTHOOK: query: FROM srcpart +INSERT OVERWRITE TABLE a PARTITION (ds='2010-03-11', hr) SELECT key, value, hr WHERE ds is not null and hr>10 +INSERT OVERWRITE TABLE b PARTITION (ds='2010-04-11', hr) SELECT key, value, hr WHERE ds is not null and hr>11 +INSERT OVERWRITE TABLE c PARTITION (ds='2010-05-11', hr) SELECT key, value, hr WHERE hr>0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=11 +POSTHOOK: Output: default@a@ds=2010-03-11/hr=12 +POSTHOOK: Output: default@b@ds=2010-04-11/hr=12 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=11 +POSTHOOK: Output: default@c@ds=2010-05-11/hr=12 +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: a PARTITION(ds=2010-03-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: b PARTITION(ds=2010-04-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: c PARTITION(ds=2010-05-11,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain select * from a +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: key (type: string), value (type: string), ds (type: string), hr (type: string) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2000 Data size: 910000 Basic stats: COMPLETE Column stats: PARTIAL + ListSink + +PREHOOK: query: explain select key from a +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from a +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: a + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 174000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select value from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 91000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from b +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from b +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: b + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE + ListSink + +PREHOOK: query: explain select value from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select value from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: value (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 182000 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select key from c +PREHOOK: type: QUERY +POSTHOOK: query: explain select key from c +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: c + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: key (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE + ListSink +