Index: ql/src/test/results/clientpositive/input_part6.q.out =================================================================== --- ql/src/test/results/clientpositive/input_part6.q.out (revision 0) +++ ql/src/test/results/clientpositive/input_part6.q.out (revision 0) @@ -0,0 +1,37 @@ +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF SRCPART x)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF x))) (TOK_WHERE (= (TOK_COLREF x ds) (- (- 2008 04) 08))) (TOK_LIMIT 10))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + x + Filter Operator + predicate: + expr: (ds = ((2008 - 4) - 8)) + type: boolean + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: ds + type: string + expr: hr + type: string + Limit + File Output Operator + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat + + Stage: Stage-0 + Fetch Operator + limit: 10 + + Index: ql/src/test/queries/clientpositive/input_part6.q =================================================================== --- ql/src/test/queries/clientpositive/input_part6.q (revision 0) +++ ql/src/test/queries/clientpositive/input_part6.q (revision 0) @@ -0,0 +1,4 @@ +EXPLAIN +SELECT x.* FROM SRCPART x WHERE x.ds = 2008-04-08 LIMIT 10; + +SELECT x.* FROM SRCPART x WHERE x.ds = 2008-04-08 LIMIT 10; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (revision 132846) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (working copy) @@ -269,15 +269,72 @@ this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("AND", this.prunerExpr, desc); } } - - /** From the table metadata prune the partitions to return the partitions **/ + + /** + * list of the partitions satisfying the pruning criteria - contains both confirmed and unknown partitions + */ + public static class PrunedPartitionList { + // confirmed partitions - satisfy the partition criteria + private Set confirmedPartns; + + // unknown partitions - may/may not satisfy the partition criteria + private Set unknownPartns; + + /** + * @param confirmedPartns confirmed paritions + * @param unknownPartns unknown partitions + */ + public PrunedPartitionList(Set confirmedPartns, Set unknownPartns) { + this.confirmedPartns = confirmedPartns; + this.unknownPartns = unknownPartns; + } + + /** + * get confirmed partitions + * @return confirmedPartns confirmed paritions + */ + public Set getConfirmedPartns() { + return confirmedPartns; + } + + /** + * get unknown partitions + * @return unknownPartns unknown paritions + */ + public Set getUnknownPartns() { + return unknownPartns; + } + + /** + * set confirmed partitions + * @param confirmedPartns confirmed paritions + */ + public void setConfirmedPartns(Set confirmedPartns) { + this.confirmedPartns = confirmedPartns; + } + + /** + * set unknown partitions + * @param unknownPartns unknown partitions + */ + public void setUnknownPartns(Set unknownPartns) { + this.unknownPartns = unknownPartns; + } + } + + /** + * From the table metadata prune the partitions to return the partitions. + * Evaluate the parition pruner for each partition and return confirmed and unknown partitions separately + */ @SuppressWarnings("nls") - public Set prune() throws HiveException { + public PrunedPartitionList prune() throws HiveException { LOG.trace("Started pruning partiton"); LOG.trace("tabname = " + this.tab.getName()); LOG.trace("prune Expression = " + this.prunerExpr); - LinkedHashSet ret_parts = new LinkedHashSet(); + LinkedHashSet true_parts = new LinkedHashSet(); + LinkedHashSet unkn_parts = new LinkedHashSet(); + try { StructObjectInspector rowObjectInspector = (StructObjectInspector)this.tab.getDeserializer().getObjectInspector(); Object[] rowWithPart = new Object[2]; @@ -311,15 +368,22 @@ if (evaluator != null) { evaluator.evaluate(rowWithPart, rowWithPartObjectInspector, inspectableObject); LOG.trace("prune result for partition " + partSpec + ": " + inspectableObject.o); - if (!Boolean.FALSE.equals(inspectableObject.o)) { + if (Boolean.TRUE.equals(inspectableObject.o)) { LOG.debug("retained partition: " + partSpec); - ret_parts.add(part); - } else { + true_parts.add(part); + } + else if (Boolean.FALSE.equals(inspectableObject.o)) { LOG.trace("pruned partition: " + partSpec); + } + else { + LOG.debug("unknown partition: " + partSpec); + unkn_parts.add(part); } } - else - ret_parts.add(part); + else { + // is there is no parition pruning, all of them are needed + true_parts.add(part); + } } } catch (Exception e) { @@ -327,7 +391,7 @@ } // Now return the set of partitions - return ret_parts; + return new PrunedPartitionList(true_parts, unkn_parts); } public Table getTable() { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 132846) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -2934,16 +2934,21 @@ if (pr.containsPartitionCols()) { List listP = new ArrayList(); List partP = new ArrayList(); + PartitionPruner.PrunedPartitionList partsList = null; Set parts = null; try { - parts = pr.prune(); - Iterator iterParts = parts.iterator(); - while (iterParts.hasNext()) { - Partition part = iterParts.next(); - listP.add(part.getPartitionPath()); - partP.add(Utilities.getPartitionDesc(part)); + partsList = pr.prune(); + // If there is any unknown partition, create a map-reduce job for the filter to prune correctly + if (partsList.getUnknownPartns().size() == 0) { + parts = partsList.getConfirmedPartns(); + Iterator iterParts = parts.iterator(); + while (iterParts.hasNext()) { + Partition part = iterParts.next(); + listP.add(part.getPartitionPath()); + partP.add(Utilities.getPartitionDesc(part)); + } + fetch = new fetchWork(listP, partP, qb.getParseInfo().getOuterQueryLimit()); } - fetch = new fetchWork(listP, partP, qb.getParseInfo().getOuterQueryLimit()); } catch (HiveException e) { // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e)); @@ -3017,7 +3022,10 @@ PartitionPruner pruner = this.aliasToPruner.get(alias_id); Set parts = null; try { - parts = pruner.prune(); + // pass both confirmed and unknown partitions through the map-reduce framework + PartitionPruner.PrunedPartitionList partsList = pruner.prune(); + parts = partsList.getConfirmedPartns(); + parts.addAll(partsList.getUnknownPartns()); } catch (HiveException e) { // Has to use full name to make sure it does not conflict with org.apache.commons.lang.StringUtils LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));