diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 8bb6d0f..2bca9fb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -699,15 +699,13 @@ private static void getTablePartitionUsedColumns(HiveOperation op, BaseSemanticA || op.equals(HiveOperation.QUERY)) { SemanticAnalyzer querySem = (SemanticAnalyzer) sem; ParseContext parseCtx = querySem.getParseContext(); - Map tsoTopMap = parseCtx.getTopToTable(); for (Map.Entry> topOpMap : querySem .getParseContext().getTopOps().entrySet()) { Operator topOp = topOpMap.getValue(); - if (topOp instanceof TableScanOperator - && tsoTopMap.containsKey(topOp)) { + if (topOp instanceof TableScanOperator) { TableScanOperator tableScanOp = (TableScanOperator) topOp; - Table tbl = tsoTopMap.get(tableScanOp); + Table tbl = tableScanOp.getConf().getTableMetadata(); List neededColumnIds = tableScanOp.getNeededColumnIDs(); List columns = tbl.getCols(); List cols = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index ed03bb9..1d61243 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -181,6 +181,8 @@ import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Shell; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; @@ -1023,7 +1025,15 @@ private static void serializeObjectByKryo(Kryo kryo, Object plan, OutputStream o private static T deserializeObjectByKryo(Kryo kryo, InputStream in, Class clazz ) { Input inp = new Input(in); + + Logger root = Logger.getRootLogger(); + if (root != null) + root.setLevel(Level.TRACE); // Log4J Level + T t = kryo.readObject(inp,clazz); + if (root != null) + root.setLevel(Level.INFO); // Log4J Level + inp.close(); return t; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java index 70c23a6..9e75ea1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java @@ -202,7 +202,6 @@ protected boolean checkConvertBucketMapJoin( new LinkedHashMap>>(); HashMap> topOps = pGraphContext.getTopOps(); - Map topToTable = pGraphContext.getTopToTable(); // (partition to bucket file names) and (partition to bucket number) for // the big table; @@ -264,7 +263,7 @@ protected boolean checkConvertBucketMapJoin( joinKeyOrder = new Integer[keys.size()]; } - Table tbl = topToTable.get(tso); + Table tbl = tso.getConf().getTableMetadata(); if (tbl.isPartitioned()) { PrunedPartitionList prunedParts = pGraphContext.getPrunedPartitions(alias, tso); List partitions = prunedParts.getNotDeniedPartns(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java index c9e8086..085ddc6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractSMBJoinProc.java @@ -249,8 +249,6 @@ private boolean isEligibleForBucketSortMergeJoin( int pos, List sortColumnsFirstTable) throws SemanticException { String alias = aliases[pos]; - Map topToTable = this.pGraphContext - .getTopToTable(); /* * Consider a query like: @@ -313,7 +311,7 @@ private boolean isEligibleForBucketSortMergeJoin( return false; } - Table tbl = topToTable.get(tso); + Table tbl = tso.getConf().getTableMetadata(); if (tbl.isPartitioned()) { PrunedPartitionList prunedParts = pGraphContext.getPrunedPartitions(alias, tso); List partitions = prunedParts.getNotDeniedPartns(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java index 8719576..843d069 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java @@ -71,7 +71,7 @@ public int getBigTablePosition(ParseContext parseCtx, JoinOperator joinOp, int numPartitions = 1; // in case the sizes match, preference is // given to the table with fewer partitions - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); long averageSize = 0; if (!table.isPartitioned()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index 2f1497a..b98d131 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -465,7 +465,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (op instanceof TableScanOperator) { assert !useBucketSortPositions; TableScanOperator ts = (TableScanOperator) op; - Table srcTable = pGraphContext.getTopToTable().get(ts); + Table srcTable = ts.getConf().getTableMetadata(); // Find the positions of the bucketed columns in the table corresponding // to the select list. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index 07093b7..8546d21 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -186,7 +186,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Obje String column = extractColName(ctx.parent); if (ts != null && column != null) { - Table table = parseContext.getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); if (table != null && table.isPartitionKey(column)) { String alias = ts.getConf().getAlias(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java index 7f574dc..0b45e25 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java @@ -70,7 +70,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx opProcCtx, TableScanOperator op = (TableScanOperator) nd; GenMRProcContext ctx = (GenMRProcContext) opProcCtx; ParseContext parseCtx = ctx.getParseCtx(); - Class inputFormat = parseCtx.getTopToTable().get(op) + Class inputFormat = op.getConf().getTableMetadata() .getInputFormatClass(); Map, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 9a74e1e..cde93fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -519,12 +519,11 @@ public static void setMapWork(MapWork plan, ParseContext parseCtx, Set props = parseCtx.getTopToProps().get(topOp); + Map props = topOp.getConf().getOpProps(); if (props != null) { Properties target = aliasPartnDesc.getProperties(); if (target == null) { @@ -946,7 +945,7 @@ public static MapredWork getMapRedWorkFromConf(HiveConf conf) { public static TableScanOperator createTemporaryTableScanOperator(RowSchema rowSchema) { TableScanOperator tableScanOp = - (TableScanOperator) OperatorFactory.get(new TableScanDesc(), rowSchema); + (TableScanOperator) OperatorFactory.get(new TableScanDesc(null), rowSchema); // Set needed columns for this dummy TableScanOperator List neededColumnIds = new ArrayList(); List neededColumnNames = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java index 603a261..c9848da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GlobalLimitOptimizer.java @@ -62,10 +62,8 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { GlobalLimitCtx globalLimitCtx = pctx.getGlobalLimitCtx(); Map opToPartPruner = pctx.getOpToPartPruner(); Map nameToSplitSample = pctx.getNameToSplitSample(); - Map topToTable = pctx.getTopToTable(); QB qb = pctx.getQB(); - HiveConf conf = pctx.getConf(); QBParseInfo qbParseInfo = qb.getParseInfo(); // determine the query qualifies reduce input size for LIMIT @@ -93,7 +91,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // query qualify for the optimization if (tempGlobalLimit != null && tempGlobalLimit != 0) { TableScanOperator ts = (TableScanOperator) topOps.values().toArray()[0]; - Table tab = topToTable.get(ts); + Table tab = ts.getConf().getTableMetadata(); if (!tab.isPartitioned()) { if (qbParseInfo.getDestToWhereExpr().isEmpty()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index 1d18e0c..1e47fcb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -304,7 +304,7 @@ protected GroupByOptimizerSortMatch checkSortGroupBy(Stack stack, // Create a mapping from the group by columns to the table columns Map tableColsMapping = new HashMap(); Set constantCols = new HashSet(); - Table table = pGraphContext.getTopToTable().get(currOp); + Table table = tableScanOp.getConf().getTableMetadata(); for (FieldSchema col : table.getAllCols()) { tableColsMapping.put(col.getName(), col.getName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java index 8207599..69ff438 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java @@ -176,7 +176,7 @@ private FetchData checkTree(boolean aggressive, ParseContext pctx, String alias, if (!aggressive && qb.hasTableSample(alias)) { return null; } - Table table = pctx.getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); if (table == null) { return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java index ea06503..4e051a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java @@ -202,9 +202,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, for (Entry> topOp : topOps.entrySet()) { TableScanOperator tso = (TableScanOperator) topOp.getValue(); - Table origTable = parseContext.getTopToTable().get(ctx.getCloneTSOpMap().get(tso)); String tabAlias = tso.getConf().getAlias(); - parseContext.getTopToTable().put(tso, origTable); int initCnt = 1; String newAlias = "subquery" + initCnt + ":" + tabAlias; while (origTopOps.containsKey(newAlias)) { @@ -410,7 +408,7 @@ private Table getTable( if (op instanceof TableScanOperator) { TableScanOperator tsOp = (TableScanOperator)op; if (tableScanOpsForJoin.contains(tsOp)) { - return parseContext.getTopToTable().get(tsOp); + return tsOp.getConf().getTableMetadata(); } } if ((op.getParentOperators() == null) || (op.getParentOperators().isEmpty()) || diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java index 6a43d1c..cf8534a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java @@ -228,7 +228,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } - Table tbl = pctx.getTopToTable().get(tsOp); + Table tbl = tsOp.getConf().getTableMetadata(); List oneRow = new ArrayList(); List ois = new ArrayList(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java index 3ca11d9..1edc5bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java @@ -60,7 +60,7 @@ public int getBigTablePosition(ParseContext parseCtx, JoinOperator joinOp, currentPos++; continue; } - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); long currentSize = 0; if (!table.isPartitioned()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java index c52f753..44fe651 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java @@ -124,7 +124,7 @@ private void findPossibleAutoConvertedJoinOperators() throws SemanticException { Set aliases = new LinkedHashSet(); for (TableScanOperator tsop : topOps) { - Table table = pCtx.getTopToTable().get(tsop); + Table table = tsop.getConf().getTableMetadata(); if (table == null) { // table should not be null. throw new SemanticException("The table of " + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java index 6020db3..06a9478 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteGBUsingIndex.java @@ -161,11 +161,10 @@ boolean shouldApplyOptimization() throws SemanticException { * if the optimization can be applied. If yes, we add the name of the top table to * the tsOpToProcess to apply rewrite later on. * */ - Map topToTable = parseContext.getTopToTable(); for (Map.Entry> entry : parseContext.getTopOps().entrySet()) { String alias = entry.getKey(); TableScanOperator topOp = (TableScanOperator) entry.getValue(); - Table table = topToTable.get(topOp); + Table table = topOp.getConf().getTableMetadata(); List indexes = tableToIndex.get(table); if (indexes.isEmpty()) { continue; @@ -232,12 +231,16 @@ private boolean checkIfRewriteCanBeApplied(String alias, TableScanOperator topOp supportedIndexes.add(AggregateIndexHandler.class.getName()); // query the metastore to know what columns we have indexed - Collection topTables = parseContext.getTopToTable().values(); + Collection> topTables = parseContext.getTopOps().values(); Map> indexes = new HashMap>(); - for (Table tbl : topTables){ - List tblIndexes = IndexUtils.getIndexes(tbl, supportedIndexes); - if (tblIndexes.size() > 0) { - indexes.put(tbl, tblIndexes); + for (Operator op : topTables) { + if (op instanceof TableScanOperator) { + TableScanOperator tsOP = (TableScanOperator) op; + List tblIndexes = IndexUtils.getIndexes(tsOP.getConf().getTableMetadata(), + supportedIndexes); + if (tblIndexes.size() > 0) { + indexes.put(tsOP.getConf().getTableMetadata(), tblIndexes); + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java index fe686d9..72f4588 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java @@ -170,7 +170,6 @@ private void replaceTableScanProcess(TableScanOperator scanOperator) throws Sema // Need to remove the original TableScanOperators from these data structures // and add new ones - Map topToTable = rewriteQueryCtx.getParseContext().getTopToTable(); Map> topOps = rewriteQueryCtx.getParseContext() .getTopOps(); Map, OpParseContext> opParseContext = rewriteQueryCtx @@ -181,13 +180,8 @@ private void replaceTableScanProcess(TableScanOperator scanOperator) throws Sema // remove original TableScanOperator topOps.remove(alias); - topToTable.remove(scanOperator); opParseContext.remove(scanOperator); - // construct a new descriptor for the index table scan - TableScanDesc indexTableScanDesc = new TableScanDesc(); - indexTableScanDesc.setGatherStats(false); - String indexTableName = rewriteQueryCtx.getIndexName(); Table indexTableHandle = null; try { @@ -198,6 +192,10 @@ private void replaceTableScanProcess(TableScanOperator scanOperator) throws Sema throw new SemanticException(e.getMessage(), e); } + // construct a new descriptor for the index table scan + TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle); + indexTableScanDesc.setGatherStats(false); + String k = indexTableName + Path.SEPARATOR; indexTableScanDesc.setStatsAggPrefix(k); scanOperator.setConf(indexTableScanDesc); @@ -227,12 +225,10 @@ private void replaceTableScanProcess(TableScanOperator scanOperator) throws Sema } // Scan operator now points to other table - topToTable.put(scanOperator, indexTableHandle); scanOperator.getConf().setAlias(newAlias); scanOperator.setAlias(indexTableName); topOps.put(newAlias, scanOperator); opParseContext.put(scanOperator, operatorContext); - rewriteQueryCtx.getParseContext().setTopToTable((HashMap) topToTable); rewriteQueryCtx.getParseContext().setTopOps( (HashMap>) topOps); rewriteQueryCtx.getParseContext().setOpParseCtx( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java index 2f517f2..d6a6ed6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java @@ -139,7 +139,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // Table scan operator. TableScanOperator top = (TableScanOperator)nd; - org.apache.hadoop.hive.ql.metadata.Table t = pctx.getTopToTable().get(top); + org.apache.hadoop.hive.ql.metadata.Table t = top.getConf().getTableMetadata(); Table tab = t.getTTable(); // Generate the mappings diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java index db16dc4..0304196 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/LBPartitionProcFactory.java @@ -52,7 +52,7 @@ protected void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, TableScanOperator top) throws SemanticException, UDFArgumentException { LBOpPartitionWalkerCtx owc = (LBOpPartitionWalkerCtx) procCtx; - Table tbl = owc.getParseContext().getTopToTable().get(top); + Table tbl = top.getConf().getTableMetadata(); if (tbl.isPartitioned()) { // Run partition pruner to get partitions ParseContext parseCtx = owc.getParseContext(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java index a8cb3c1..4e49260 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java @@ -161,7 +161,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator ts = (TableScanOperator)nd; AnnotateOpTraitsProcCtx opTraitsCtx = (AnnotateOpTraitsProcCtx)procCtx; - Table table = opTraitsCtx.getParseContext().getTopToTable().get(ts); + Table table = ts.getConf().getTableMetadata(); PrunedPartitionList prunedPartList = null; try { prunedPartList = diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java index d517ab6..ae96def 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java @@ -28,6 +28,7 @@ import java.util.Stack; import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler; @@ -47,6 +48,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.MapredWork; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; /** * @@ -115,12 +117,14 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) supportedIndexes.add(BitmapIndexHandler.class.getName()); // query the metastore to know what columns we have indexed - Collection
topTables = pctx.getTopToTable().values(); Map> indexes = new HashMap>(); - for (Map.Entry entry : pctx.getTopToTable().entrySet()) { - List tblIndexes = IndexUtils.getIndexes(entry.getValue(), supportedIndexes); - if (tblIndexes.size() > 0) { - indexes.put(entry.getKey(), tblIndexes); + for (Operator op : pctx.getTopOps().values()) { + if (op instanceof TableScanOperator) { + List tblIndexes = IndexUtils.getIndexes(((TableScanOperator) op).getConf() + .getTableMetadata(), supportedIndexes); + if (tblIndexes.size() > 0) { + indexes.put((TableScanOperator) op, tblIndexes); + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 4b2a81a..1708e1b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -138,7 +138,7 @@ public static boolean onlyContainsPartnCols(Table tab, ExprNodeDesc expr) { */ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx, String alias) throws SemanticException { - return prune(parseCtx.getTopToTable().get(ts), parseCtx.getOpToPartPruner().get(ts), + return prune(ts.getConf().getTableMetadata(), parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias, parseCtx.getPrunedPartitions()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index 8572491..8bba7b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -103,7 +103,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; PrunedPartitionList partList = aspCtx.getParseContext().getPrunedPartitions(tsop.getName(), tsop); - Table table = aspCtx.getParseContext().getTopToTable().get(tsop); + Table table = tsop.getConf().getTableMetadata(); try { // gather statistics for the first time and the attach it to table scan operator diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java index f36f87f..c2286a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java @@ -17,16 +17,20 @@ */ package org.apache.hadoop.hive.ql.parse; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; public class ColumnAccessAnalyzer { - private static final Log LOG = LogFactory.getLog(ColumnAccessAnalyzer.class.getName()); + private static final Log LOG = LogFactory.getLog(ColumnAccessAnalyzer.class.getName()); private final ParseContext pGraphContext; public ColumnAccessAnalyzer() { @@ -39,19 +43,22 @@ public ColumnAccessAnalyzer(ParseContext pactx) { public ColumnAccessInfo analyzeColumnAccess() throws SemanticException { ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo(); - Map topOps = pGraphContext.getTopToTable(); - for (TableScanOperator op : topOps.keySet()) { - Table table = topOps.get(op); - String tableName = table.getCompleteName(); - List referenced = op.getReferencedColumns(); - for (String column : referenced) { - columnAccessInfo.add(tableName, column); - } - if (table.isPartitioned()) { - PrunedPartitionList parts = pGraphContext.getPrunedPartitions(table.getTableName(), op); - if (parts.getReferredPartCols() != null) { - for (String partKey : parts.getReferredPartCols()) { - columnAccessInfo.add(tableName, partKey); + Collection> topOps = pGraphContext.getTopOps().values(); + for (Operator op : topOps) { + if (op instanceof TableScanOperator) { + TableScanOperator top = (TableScanOperator) op; + Table table = top.getConf().getTableMetadata(); + String tableName = table.getCompleteName(); + List referenced = top.getReferencedColumns(); + for (String column : referenced) { + columnAccessInfo.add(tableName, column); + } + if (table.isPartitioned()) { + PrunedPartitionList parts = pGraphContext.getPrunedPartitions(table.getTableName(), top); + if (parts.getReferredPartCols() != null) { + for (String partKey : parts.getReferredPartCols()) { + columnAccessInfo.add(tableName, partKey); + } } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java index 627ca7b..8a50249 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java @@ -191,10 +191,7 @@ public MapWork createMapWork(GenTezProcContext context, Operator root, setupMapWork(mapWork, context, partitions, root, alias); - if (context.parseContext != null - && context.parseContext.getTopToTable() != null - && context.parseContext.getTopToTable().containsKey(ts) - && context.parseContext.getTopToTable().get(ts).isDummyTable()) { + if (ts.getConf().getTableMetadata() != null && ts.getConf().getTableMetadata().isDummyTable()) { mapWork.setDummyTableScan(true); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 8215c26..872a5fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -73,15 +73,12 @@ private HashMap opToSamplePruner; private Map> opToPartToSkewedPruner; private HashMap> topOps; - private HashMap> topSelOps; private LinkedHashMap, OpParseContext> opParseCtx; private Map joinContext; private Map mapJoinContext; private Map smbMapJoinContext; - private HashMap topToTable; private Map fsopToTable; private List reduceSinkOperatorsAddedByEnforceBucketingSorting; - private HashMap> topToProps; private HashMap nameToSplitSample; private List loadTableWork; private List loadFileWork; @@ -128,15 +125,11 @@ public ParseContext() { * @param opToPartList * @param topOps * list of operators for the top query - * @param topSelOps - * list of operators for the selects introduced for column pruning * @param opParseCtx * operator parse context - contains a mapping from operator to * operator parse state (row resolver etc.) * @param joinContext * context needed join processing (map join specifically) - * @param topToTable - * the top tables being processed * @param loadTableWork * list of destination tables being loaded * @param loadFileWork @@ -163,12 +156,9 @@ public ParseContext( HashMap opToPartPruner, HashMap opToPartList, HashMap> topOps, - HashMap> topSelOps, LinkedHashMap, OpParseContext> opParseCtx, Map joinContext, Map smbMapJoinContext, - HashMap topToTable, - HashMap> topToProps, Map fsopToTable, List loadTableWork, List loadFileWork, Context ctx, HashMap idToTableNameMap, int destTableId, @@ -190,14 +180,11 @@ public ParseContext( this.opToPartList = opToPartList; this.joinContext = joinContext; this.smbMapJoinContext = smbMapJoinContext; - this.topToTable = topToTable; this.fsopToTable = fsopToTable; - this.topToProps = topToProps; this.loadFileWork = loadFileWork; this.loadTableWork = loadTableWork; this.opParseCtx = opParseCtx; this.topOps = topOps; - this.topSelOps = topSelOps; this.ctx = ctx; this.idToTableNameMap = idToTableNameMap; this.destTableId = destTableId; @@ -297,21 +284,6 @@ public void setOpToPartPruner( return opToPartList; } - /** - * @return the topToTable - */ - public HashMap getTopToTable() { - return topToTable; - } - - /** - * @param topToTable - * the topToTable to set - */ - public void setTopToTable(HashMap topToTable) { - this.topToTable = topToTable; - } - public Map getFsopToTable() { return fsopToTable; } @@ -331,21 +303,6 @@ public void setReduceSinkOperatorsAddedByEnforceBucketingSorting( } /** - * @return the topToProps - */ - public HashMap> getTopToProps() { - return topToProps; - } - - /** - * @param topToProps - * the topToProps to set - */ - public void setTopToProps(HashMap> topToProps) { - this.topToProps = topToProps; - } - - /** * @return the topOps */ public HashMap> getTopOps() { @@ -361,22 +318,6 @@ public void setTopOps(HashMap> topOps) } /** - * @return the topSelOps - */ - public HashMap> getTopSelOps() { - return topSelOps; - } - - /** - * @param topSelOps - * the topSelOps to set - */ - public void setTopSelOps( - HashMap> topSelOps) { - this.topSelOps = topSelOps; - } - - /** * @return the opParseCtx */ public LinkedHashMap, OpParseContext> getOpParseCtx() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index 9fcc1b2..2ecf8a5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@ -75,7 +75,7 @@ public Object process(Node nd, Stack stack, TableScanOperator tableScan = (TableScanOperator) nd; ParseContext parseContext = context.parseContext; - Class inputFormat = parseContext.getTopToTable().get(tableScan) + Class inputFormat = tableScan.getConf().getTableMetadata() .getInputFormatClass(); QB queryBlock = parseContext.getQB(); QBParseInfo parseInfo = parseContext.getQB().getParseInfo(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c2d5c8c..0262909 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -374,7 +374,6 @@ public void initParseCtx(ParseContext pctx) { opToPartList = pctx.getOpToPartList(); opToSamplePruner = pctx.getOpToSamplePruner(); topOps = pctx.getTopOps(); - topSelOps = pctx.getTopSelOps(); opParseCtx = pctx.getOpParseCtx(); loadTableWork = pctx.getLoadTableWork(); loadFileWork = pctx.getLoadFileWork(); @@ -393,15 +392,12 @@ public void initParseCtx(ParseContext pctx) { } public ParseContext getParseContext() { - return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList, topOps, - topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable, topToTableProps, - fsopToTable, loadTableWork, - loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, - opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, + return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList, topOps, opParseCtx, + joinContext, smbMapJoinContext, fsopToTable, loadTableWork, loadFileWork, ctx, + idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, + prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, viewAliasToInput, - reduceSinkOperatorsAddedByEnforceBucketingSorting, - queryProperties); + reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); } @SuppressWarnings("nls") @@ -9272,7 +9268,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { } // Create the root of the operator tree - TableScanDesc tsDesc = new TableScanDesc(alias, vcList); + TableScanDesc tsDesc = new TableScanDesc(alias, vcList, tab); setupStats(tsDesc, qb.getParseInfo(), tab, alias, rwsch); SplitSample sample = nameToSplitSample.get(alias_id); @@ -9294,6 +9290,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { Map props = qb.getTabPropsForAlias(alias); if (props != null) { topToTableProps.put((TableScanOperator) top, props); + tsDesc.setOpProps(props); } } else { rwsch = opParseCtx.get(top).getRowResolver(); @@ -9987,12 +9984,12 @@ void analyzeInternal(ASTNode ast, PlannerContext plannerCtx) throws SemanticExce } // 4. Generate Parse Context for Optimizer & Physical compiler - ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child, opToPartPruner, opToPartList, - topOps, topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable, topToTableProps, - fsopToTable, loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx, - listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions, opToSamplePruner, - globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner, - viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); + ParseContext pCtx = new ParseContext(conf, qb, plannerCtx.child, opToPartPruner, opToPartList, topOps, + opParseCtx, joinContext, smbMapJoinContext, fsopToTable, loadTableWork, loadFileWork, ctx, + idToTableNameMap, destTableId, uCtx, listMapJoinOpsNoReducer, groupOpToInputTables, + prunedPartitions, opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks, + opToPartToSkewedPruner, viewAliasToInput, + reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties); // 5. Take care of view creation if (createVwDesc != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java index da14ab4..77e624d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java @@ -143,7 +143,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // Must be deterministic order map for consistent q-test output across Java versions Map> tableToKeysMap = new LinkedHashMap>(); - Table tbl = pGraphContext.getTopToTable().get(tso); + Table tbl = tso.getConf().getTableMetadata(); tableToKeysMap.put(tbl.getCompleteName(), keyColNames); tableAccessCtx.addOperatorTableAccess(op, tableToKeysMap); @@ -203,7 +203,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, return null; } - Table tbl = pGraphContext.getTopToTable().get(tso); + Table tbl = tso.getConf().getTableMetadata(); tableToKeysMap.put(tbl.getCompleteName(), keyColNames); } else { return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 23fbbe1..05cf757 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -386,8 +386,8 @@ public ParseContext getParseContext(ParseContext pCtx, List opProps; + @Override @Explain(skipHeader = true, displayName = "Statistics") @@ -51,4 +55,12 @@ public OpTraits getOpTraits() { public void setOpTraits(OpTraits opTraits) { this.opTraits = opTraits; } + + public Map getOpProps() { + return opProps; + } + + public void setOpProps(Map props) { + this.opProps = props; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java index c8c9570..c39a8d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.io.Serializable; +import java.util.Map; public interface OperatorDesc extends Serializable, Cloneable { public Object clone() throws CloneNotSupportedException; @@ -26,4 +27,5 @@ public void setStatistics(Statistics statistics); public OpTraits getOpTraits(); public void setOpTraits(OpTraits opTraits); + public Map getOpProps(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index ae99bd3..b62ffed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -956,7 +956,7 @@ public static void addInputsForView(ParseContext parseCtx) throws HiveException ReadEntity parentViewInfo = getParentViewInfo(alias, parseCtx.getViewAliasToInput()); // Adds tables only for create view (PPD filter can be appended by outer query) - Table table = parseCtx.getTopToTable().get(topOp); + Table table = topOp.getConf().getTableMetadata(); PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 699b476..bb9d41f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; /** @@ -93,23 +94,27 @@ private boolean isMetadataOnly = false; + private transient final Table tblMetadata; + @SuppressWarnings("nls") - public TableScanDesc() { + public TableScanDesc(Table tblMetadata) { + this(null, tblMetadata); } - public TableScanDesc(final String alias) { - this.alias = alias; + public TableScanDesc(final String alias, Table tblMetadata) { + this(alias, null, tblMetadata); } - public TableScanDesc(final String alias, List vcs) { + public TableScanDesc(final String alias, List vcs, Table tblMetadata) { this.alias = alias; this.virtualCols = vcs; + this.tblMetadata = tblMetadata; } @Override public Object clone() { List vcs = new ArrayList(getVirtualCols()); - return new TableScanDesc(getAlias(), vcs); + return new TableScanDesc(getAlias(), vcs, this.tblMetadata); } @Explain(displayName = "alias") @@ -250,8 +255,12 @@ public void setBucketFileNameMapping(Map bucketFileNameMapping) public void setIsMetadataOnly(boolean metadata_only) { isMetadataOnly = metadata_only; } - + public boolean getIsMetadataOnly() { return isMetadataOnly; } + + public Table getTableMetadata() { + return tblMetadata; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java index 576b8f0..0a7a757 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java @@ -897,7 +897,7 @@ private static ExprNodeGenericFuncDesc pushFilterToStorageHandler( HiveConf hiveConf) { TableScanDesc tableScanDesc = tableScanOp.getConf(); - Table tbl = owi.getParseContext().getTopToTable().get(tableScanOp); + Table tbl = tableScanDesc.getTableMetadata(); if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER)) { // attach the original predicate to the table scan operator for index // optimizations that require the pushed predicate before pcr & later diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java index 5871ad7..d9ab9c0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestGenTezWork.java @@ -95,7 +95,7 @@ protected void setupMapWork(MapWork mapWork, GenTezProcContext context, rs = new ReduceSinkOperator(); rs.setConf(new ReduceSinkDesc()); ts = new TableScanOperator(); - ts.setConf(new TableScanDesc()); + ts.setConf(new TableScanDesc(null)); ts.getChildOperators().add(rs); rs.getParentOperators().add(ts); rs.getChildOperators().add(fs);