diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 3ca2fad..af69e86 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -872,7 +872,7 @@ HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false), // Whether to generate the splits locally or in the AM (tez only) - HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true), + HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true), // none, idonly, traverse, execution HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none"), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java index b3d86b0..5ee16f7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java @@ -34,8 +34,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java index 1ccbb34..b0b0925 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java @@ -27,11 +27,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.DemuxDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.Deserializer; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java index 06eb5b8..0e3cfe7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java @@ -27,12 +27,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; -import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index c627684..cb1cbf5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3163,59 +3163,6 @@ private static void createTmpDirs(Configuration conf, } } - public static long getSize(String alias, Table table, HiveConf conf, - TableScanOperator topOp, ExprNodeDesc expr) throws HiveException { - long result = 0; - int numPartitions = 0; - Map prunedPartitionsMap - = new HashMap(); - - if (!table.isPartitioned()) { - result = getSize(conf, table); - } - else { - // For partitioned tables, get the size of all the partitions - PrunedPartitionList partsList = PartitionPruner.prune(table, expr, conf, - alias, prunedPartitionsMap); - numPartitions = partsList.getNotDeniedPartns().size(); - for (Partition part : partsList.getNotDeniedPartns()) { - result += getSize(conf, part); - } - } - return result; - } - - private static long getSize(HiveConf conf, String size, Path path) { - // If the size is present in the metastore, use it - if (size != null) { - try { - return Long.valueOf(size); - } catch (NumberFormatException e) { - return -1; - } - } - - try { - FileSystem fs = path.getFileSystem(conf); - return fs.getContentSummary(path).getLength(); - } catch (Exception e) { - return -1; - } - } - - private static long getSize(HiveConf conf, Table table) { - Path path = table.getPath(); - String size = table.getProperty("totalSize"); - return getSize(conf, size, path); - } - - private static long getSize(HiveConf conf, Partition partition) { - Path path = partition.getPartitionPath(); - String size = partition.getParameters().get("totalSize"); - - return getSize(conf, size, path); - } - public static void clearWorkMap() { gWorkMap.clear(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java index 0004a0e..51464e5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java @@ -125,9 +125,6 @@ protected void addPruningPred(Map opToPrunner, // Put the mapping from table scan operator to pruner_pred opToPrunner.put(top, pruner_pred); - // Set the predicate in the table directly - top.getConf().setPruningPredicate(pruner_pred); - return; } @@ -168,9 +165,6 @@ protected void addPruningPred(Map> // Put the mapping from table scan operator to part-pruner map opToPrunner.put(top, partToPruner); - // Set the predicate in the table directly - top.getConf().setPruningPredicate(pruner_pred); - return; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 2ef4881..6bdf394 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -155,7 +155,7 @@ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parse * pruner condition. * @throws HiveException */ - public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr, + private static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr, HiveConf conf, String alias, Map prunedPartitionsMap) throws HiveException { LOG.trace("Started pruning partiton"); diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 387c376..a92d4f5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -8352,9 +8352,6 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // Add a mapping from the table scan operator to Table topToTable.put((TableScanOperator) top, tab); - // set the table in the tablescan descriptor directly - ((TableScanOperator) top).getConf().setTable(tab); - Map props = qb.getTabPropsForAlias(alias); if (props != null) { topToTableProps.put((TableScanOperator) top, props); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java index 0f61713..9c35890 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java @@ -22,8 +22,6 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.PTFUtils; -import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; /** @@ -54,10 +52,6 @@ */ private int rowLimit = -1; - private transient Table table; - - private transient ExprNodeDesc pruningPredicate; - /** * A boolean variable set to true by the semantic analyzer only in case of the analyze command. * @@ -77,32 +71,10 @@ // input file name (big) to bucket number private Map bucketFileNameMapping; - static{ - PTFUtils.makeTransient(TableScanDesc.class, "pruningPredicate"); - PTFUtils.makeTransient(TableScanDesc.class, "table"); - } - - @SuppressWarnings("nls") public TableScanDesc() { } - public Table getTable() { - return table; - } - - public void setTable(Table t) { - table = t; - } - - public ExprNodeDesc getPruningPredicate() { - return pruningPredicate; - } - - public void setPruningPredicate(ExprNodeDesc expr) { - pruningPredicate = expr; - } - public TableScanDesc(final String alias) { this.alias = alias; }