Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision 1083890) +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (working copy) @@ -269,7 +269,7 @@ public Partition getPartition(String dbName, String tblName, String name) throws MetaException, UnknownTableException, NoSuchObjectException, TException; - + /** * @param dbName * @param tableName @@ -285,7 +285,7 @@ public Partition getPartitionWithAuthInfo(String dbName, String tableName, List pvals, String userName, List groupNames) throws MetaException, UnknownTableException, NoSuchObjectException, TException; - + /** * @param tbl_name * @param db_name @@ -308,13 +308,31 @@ List part_vals, short max_parts) throws MetaException, TException; /** + * Get list of partitions matching specified filter + * @param db_name the database name + * @param tbl_name the table name + * @param filter the filter string, + * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can + * be done only on string partition keys. + * @param max_parts the maximum number of partitions to return, + * all partitions are returned if -1 is passed + * @return list of partitions + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + public List listPartitionsByFilter(String db_name, String tbl_name, + String filter, short max_parts) throws MetaException, + NoSuchObjectException, TException; + + /** * @param dbName * @param tableName * @param s * @param userName * @param groupNames * @return - * @throws NoSuchObjectException + * @throws NoSuchObjectException */ public List listPartitionsWithAuthInfo(String dbName, String tableName, short s, String userName, List groupNames) @@ -328,7 +346,7 @@ * @param userName * @param groupNames * @return - * @throws NoSuchObjectException + * @throws NoSuchObjectException */ public List listPartitionsWithAuthInfo(String dbName, String tableName, List partialPvals, short s, String userName, @@ -531,7 +549,7 @@ public boolean dropIndex(String db_name, String tbl_name, String name, boolean deleteData) throws NoSuchObjectException, MetaException, TException; - + /** * @param Role * role object @@ -545,8 +563,8 @@ /** * @param role_name * role name - * @param db_name - * + * @param db_name + * * @return * @throws MetaException * @throws TException @@ -556,13 +574,13 @@ /** * list all role names * @return - * @throws TException - * @throws MetaException + * @throws TException + * @throws MetaException */ public List listRoleNames() throws MetaException, TException; /** - * + * * @param role_name * @param user_name * @param principalType @@ -584,7 +602,7 @@ * user name * @param principalType * @param db_name - * + * * @return * @throws MetaException * @throws TException @@ -593,7 +611,7 @@ PrincipalType principalType) throws MetaException, TException; /** - * + * * @param principalName * @param principalType * @return @@ -614,7 +632,7 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectRef hiveObject, String user_name, List group_names) throws MetaException, TException; - + /** * @param principal_name * @param principal_type Index: metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (revision 1083890) +++ metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java (working copy) @@ -23,11 +23,11 @@ import org.antlr.runtime.ANTLRStringStream; import org.antlr.runtime.CharStream; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.serde.Constants; -import org.apache.hadoop.hive.common.FileUtils; /** * The Class representing the filter as a binary tree. The tree has TreeNode's @@ -239,19 +239,30 @@ // makePartName with one key will return a substring of the name made // with both all the keys. String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false); - + StringBuilder fltr = new StringBuilder(); if (keyCount == 1) { // Case where this is no other partition columns params.put(paramName, escapedNameFragment); + fltr.append("partitionName == ").append(paramName); } else if (keyPos + 1 == keyCount) { // Case where the partition column is at the end of the name. There will // be a leading '/' but no trailing '/' - params.put(paramName, ".*/" + escapedNameFragment); + params.put(paramName, "/" + escapedNameFragment); + fltr.append("partitionName.endsWith(").append(paramName).append(')'); + } else if (keyPos == 0) { + // Case where the parttion column is at the beginning of the name. There will + // be a trailing '/' but no leading '/' + params.put(paramName, escapedNameFragment + "/"); + fltr.append("partitionName.startsWith(").append(paramName).append(')'); } else { - params.put(paramName, ".*" + escapedNameFragment + "/.*"); + // Case where the partition column is in the middle of the name. There will + // be a leading '/' and an trailing '/' + params.put(paramName, "/" + escapedNameFragment + "/"); + fltr.append("partitionName.indexOf(").append(paramName).append(") >= 0"); } - return "partitionName.matches(" + paramName + ")"; + return fltr.toString(); } + /** * The root node for the tree. */ Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1083890) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -1221,6 +1221,7 @@ Query query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2"); query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.setOrdering("partitionName ascending"); mparts = (List) query.execute(tableName, dbName); LOG.debug("Done executing query for listMPartitions"); pm.retrieveAll(mparts); @@ -1240,6 +1241,7 @@ openTransaction(); List parts = convertToParts(listMPartitionsByFilter(dbName, tblName, filter, maxParts)); + LOG.info("# parts after pruning = " + parts.size()); commitTransaction(); return parts; } @@ -1269,6 +1271,7 @@ } String jdoFilter = parser.tree.generateJDOFilter(table, params); + LOG.debug("jdoFilter = " + jdoFilter); if( jdoFilter.trim().length() > 0 ) { queryBuilder.append(" && ( "); @@ -1276,7 +1279,6 @@ queryBuilder.append(" )"); } } - return queryBuilder.toString(); } @@ -1347,7 +1349,7 @@ List partNames = new ArrayList(); try { openTransaction(); - LOG.debug("Executing listMPartitionsByFilter"); + LOG.debug("Executing listMPartitionNamesByFilter"); dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); Index: ql/src/test/results/clientpositive/ppr_pushdown2.q.out =================================================================== --- ql/src/test/results/clientpositive/ppr_pushdown2.q.out (revision 0) +++ ql/src/test/results/clientpositive/ppr_pushdown2.q.out (revision 0) @@ -0,0 +1,274 @@ +PREHOOK: query: create table ppr_test (key string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ppr_test (key string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ppr_test +PREHOOK: query: insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=2 +POSTHOOK: query: insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=2 +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=22 +POSTHOOK: query: insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=22 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +PREHOOK: query: select * from ppr_test where ds = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=2 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_191_4798709120900823624/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=2 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_191_4798709120900823624/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +2 2 +PREHOOK: query: select * from ppr_test where ds = '22' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=22 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_756_2735541929297276723/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '22' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=22 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-37-54_756_2735541929297276723/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +22 22 +PREHOOK: query: create table ppr_test2 (key string) partitioned by (ds string, s string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ppr_test2 (key string) partitioned by (ds string, s string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ppr_test2 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test2@ds=1/s=2 +POSTHOOK: query: insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test2@ds=1/s=2 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test2@ds=2/s=1 +POSTHOOK: query: insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test2@ds=2/s=1 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +PREHOOK: query: select * from ppr_test2 where s = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test2@ds=2/s=1 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-03_996_5327659435572703549/-mr-10000 +POSTHOOK: query: select * from ppr_test2 where s = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test2@ds=2/s=1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-03_996_5327659435572703549/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +2 2 1 +PREHOOK: query: select * from ppr_test2 where ds = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test2@ds=1/s=2 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-04_281_8196775271555582839/-mr-10000 +POSTHOOK: query: select * from ppr_test2 where ds = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test2@ds=1/s=2 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-04_281_8196775271555582839/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +1 1 2 +PREHOOK: query: create table ppr_test3 (key string) partitioned by (col string, ol string, l string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ppr_test3 (key string) partitioned by (col string, ol string, l string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ppr_test3 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test3@col=1/ol=2/l=3 +POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test3@col=1/ol=2/l=3 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test3@col=1/ol=1/l=2 +POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test3@col=1/ol=1/l=2 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +PREHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: query: insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +PREHOOK: query: select * from ppr_test3 where l = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_246_6425170733915100693/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where l = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_246_6425170733915100693/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +3 1 2 1 +PREHOOK: query: select * from ppr_test3 where l = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_523_447214062612676127/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where l = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_523_447214062612676127/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +2 1 1 2 +PREHOOK: query: select * from ppr_test3 where ol = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_813_6088158785291097781/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where ol = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-17_813_6088158785291097781/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +2 1 1 2 +PREHOOK: query: select * from ppr_test3 where ol = '2' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=3 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_142_6933403927924739851/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where ol = '2' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=3 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_142_6933403927924739851/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +3 1 2 1 +1 1 2 3 +PREHOOK: query: select * from ppr_test3 where col = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=3 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_602_2485861984482150436/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where col = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=1/l=2 +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=3 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-18_602_2485861984482150436/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +2 1 1 2 +3 1 2 1 +1 1 2 3 +PREHOOK: query: select * from ppr_test3 where ol = '2' and l = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_214_5779339277041051549/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where ol = '2' and l = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_214_5779339277041051549/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +3 1 2 1 +PREHOOK: query: select * from ppr_test3 where col='1' and ol = '2' and l = '1' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_526_2052468333243542612/-mr-10000 +POSTHOOK: query: select * from ppr_test3 where col='1' and ol = '2' and l = '1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test3@col=1/ol=2/l=1 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-16_14-38-19_526_2052468333243542612/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=22).key SIMPLE [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=1,s=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test2 PARTITION(ds=2,s=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=1,l=2).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=1).key SIMPLE [] +POSTHOOK: Lineage: ppr_test3 PARTITION(col=1,ol=2,l=3).key SIMPLE [] +3 1 2 1 Index: ql/src/test/results/clientpositive/ppr_pushdown.q.out =================================================================== --- ql/src/test/results/clientpositive/ppr_pushdown.q.out (revision 0) +++ ql/src/test/results/clientpositive/ppr_pushdown.q.out (revision 0) @@ -0,0 +1,455 @@ +PREHOOK: query: create table ppr_test (key string) partitioned by (ds string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ppr_test (key string) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ppr_test +PREHOOK: query: alter table ppr_test add partition (ds = '1234') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '1234') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=1234 +PREHOOK: query: alter table ppr_test add partition (ds = '1224') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '1224') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=1224 +PREHOOK: query: alter table ppr_test add partition (ds = '1214') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '1214') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=1214 +PREHOOK: query: alter table ppr_test add partition (ds = '12+4') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '12+4') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=12+4 +PREHOOK: query: alter table ppr_test add partition (ds = '12.4') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '12.4') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=12.4 +PREHOOK: query: alter table ppr_test add partition (ds = '12:4') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '12:4') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=12%3A4 +PREHOOK: query: alter table ppr_test add partition (ds = '12%4') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '12%4') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=12%254 +PREHOOK: query: alter table ppr_test add partition (ds = '12*4') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ppr_test +POSTHOOK: query: alter table ppr_test add partition (ds = '12*4') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ppr_test +POSTHOOK: Output: default@ppr_test@ds=12%2A4 +PREHOOK: query: insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=1234 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=1234 +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=1224 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=1224 +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=1214 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=1214 +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=12+4 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=12+4 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=12.4 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=12.4 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=12%3A4 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=12%3A4 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=12%254 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=12%254 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +PREHOOK: query: insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@ppr_test@ds=12%2A4 +POSTHOOK: query: insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@ppr_test@ds=12%2A4 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +PREHOOK: query: select * from ppr_test where ds = '1234' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1234 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-53_994_7562932232779987377/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1234' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1234 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-53_994_7562932232779987377/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 1234 +1234 1234 +PREHOOK: query: select * from ppr_test where ds = '1224' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1224 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_581_2505144516732828104/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1224' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1224 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_581_2505144516732828104/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 1224 +1224 1224 +PREHOOK: query: select * from ppr_test where ds = '1214' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1214 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_853_7951716559154890872/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1214' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1214 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-54_853_7951716559154890872/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 1214 +1214 1214 +PREHOOK: query: select * from ppr_test where ds = '12.4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12.4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_133_1358900232060245493/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12.4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12.4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_133_1358900232060245493/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 12.4 +12.4 12.4 +PREHOOK: query: select * from ppr_test where ds = '12+4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12+4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_396_1886989364983202450/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12+4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12+4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_396_1886989364983202450/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 12+4 +12+4 12+4 +PREHOOK: query: select * from ppr_test where ds = '12:4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%3A4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_675_1879183848037008858/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12:4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%3A4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_675_1879183848037008858/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 12:4 +12:4 12:4 +PREHOOK: query: select * from ppr_test where ds = '12%4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%254 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_947_7257143082337477962/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12%4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%254 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-55_947_7257143082337477962/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 12%4 +12%4 12%4 +PREHOOK: query: select * from ppr_test where ds = '12*4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%2A4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_206_658052552869274889/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12*4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%2A4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_206_658052552869274889/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +abcd 12*4 +12*4 12*4 +PREHOOK: query: select * from ppr_test where ds = '12.*4' +PREHOOK: type: QUERY +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_473_2360145199088935052/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12.*4' +POSTHOOK: type: QUERY +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_473_2360145199088935052/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +PREHOOK: query: select * from ppr_test where ds = '1234' and key = '1234' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1234 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_533_2669295483949497025/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1234' and key = '1234' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1234 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-56_533_2669295483949497025/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +1234 1234 +PREHOOK: query: select * from ppr_test where ds = '1224' and key = '1224' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1224 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-59_506_7569310480938065041/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1224' and key = '1224' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1224 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-42-59_506_7569310480938065041/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +1224 1224 +PREHOOK: query: select * from ppr_test where ds = '1214' and key = '1214' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=1214 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-02_471_7943670690328696677/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '1214' and key = '1214' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=1214 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-02_471_7943670690328696677/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +1214 1214 +PREHOOK: query: select * from ppr_test where ds = '12.4' and key = '12.4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12.4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-05_426_6599608597912720244/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12.4' and key = '12.4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12.4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-05_426_6599608597912720244/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +12.4 12.4 +PREHOOK: query: select * from ppr_test where ds = '12+4' and key = '12+4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12+4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-08_327_2168484222242026552/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12+4' and key = '12+4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12+4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-08_327_2168484222242026552/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +12+4 12+4 +PREHOOK: query: select * from ppr_test where ds = '12:4' and key = '12:4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%3A4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-11_361_8648952567109428131/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12:4' and key = '12:4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%3A4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-11_361_8648952567109428131/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +12:4 12:4 +PREHOOK: query: select * from ppr_test where ds = '12%4' and key = '12%4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%254 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-14_254_1733336023126589571/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12%4' and key = '12%4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%254 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-14_254_1733336023126589571/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +12%4 12%4 +PREHOOK: query: select * from ppr_test where ds = '12*4' and key = '12*4' +PREHOOK: type: QUERY +PREHOOK: Input: default@ppr_test@ds=12%2A4 +PREHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-17_286_8476862252367771252/-mr-10000 +POSTHOOK: query: select * from ppr_test where ds = '12*4' and key = '12*4' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ppr_test@ds=12%2A4 +POSTHOOK: Output: file:/tmp/nzhang/hive_2011-03-15_17-43-17_286_8476862252367771252/-mr-10000 +POSTHOOK: Lineage: ppr_test PARTITION(ds=12%4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12*4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12+4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12.4).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1214).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1224).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=1234).key EXPRESSION [] +POSTHOOK: Lineage: ppr_test PARTITION(ds=12:4).key EXPRESSION [] +12*4 12*4 Index: ql/src/test/queries/clientpositive/ppr_pushdown.q =================================================================== --- ql/src/test/queries/clientpositive/ppr_pushdown.q (revision 0) +++ ql/src/test/queries/clientpositive/ppr_pushdown.q (revision 0) @@ -0,0 +1,41 @@ +create table ppr_test (key string) partitioned by (ds string); + +alter table ppr_test add partition (ds = '1234'); +alter table ppr_test add partition (ds = '1224'); +alter table ppr_test add partition (ds = '1214'); +alter table ppr_test add partition (ds = '12+4'); +alter table ppr_test add partition (ds = '12.4'); +alter table ppr_test add partition (ds = '12:4'); +alter table ppr_test add partition (ds = '12%4'); +alter table ppr_test add partition (ds = '12*4'); + +insert overwrite table ppr_test partition(ds = '1234') select * from (select '1234' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '1224') select * from (select '1224' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '1214') select * from (select '1214' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '12+4') select * from (select '12+4' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '12.4') select * from (select '12.4' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '12:4') select * from (select '12:4' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '12%4') select * from (select '12%4' from src limit 1 union all select 'abcd' from src limit 1) s; +insert overwrite table ppr_test partition(ds = '12*4') select * from (select '12*4' from src limit 1 union all select 'abcd' from src limit 1) s; + + +select * from ppr_test where ds = '1234'; +select * from ppr_test where ds = '1224'; +select * from ppr_test where ds = '1214'; +select * from ppr_test where ds = '12.4'; +select * from ppr_test where ds = '12+4'; +select * from ppr_test where ds = '12:4'; +select * from ppr_test where ds = '12%4'; +select * from ppr_test where ds = '12*4'; +select * from ppr_test where ds = '12.*4'; + +select * from ppr_test where ds = '1234' and key = '1234'; +select * from ppr_test where ds = '1224' and key = '1224'; +select * from ppr_test where ds = '1214' and key = '1214'; +select * from ppr_test where ds = '12.4' and key = '12.4'; +select * from ppr_test where ds = '12+4' and key = '12+4'; +select * from ppr_test where ds = '12:4' and key = '12:4'; +select * from ppr_test where ds = '12%4' and key = '12%4'; +select * from ppr_test where ds = '12*4' and key = '12*4'; + + Index: ql/src/test/queries/clientpositive/ppr_pushdown2.q =================================================================== --- ql/src/test/queries/clientpositive/ppr_pushdown2.q (revision 0) +++ ql/src/test/queries/clientpositive/ppr_pushdown2.q (revision 0) @@ -0,0 +1,29 @@ +create table ppr_test (key string) partitioned by (ds string); + +insert overwrite table ppr_test partition(ds='2') select '2' from src limit 1; +insert overwrite table ppr_test partition(ds='22') select '22' from src limit 1; + +select * from ppr_test where ds = '2'; +select * from ppr_test where ds = '22'; + + +create table ppr_test2 (key string) partitioned by (ds string, s string); +insert overwrite table ppr_test2 partition(ds='1', s='2') select '1' from src limit 1; +insert overwrite table ppr_test2 partition(ds='2', s='1') select '2' from src limit 1; + +select * from ppr_test2 where s = '1'; +select * from ppr_test2 where ds = '1'; + + +create table ppr_test3 (key string) partitioned by (col string, ol string, l string); +insert overwrite table ppr_test3 partition(col='1', ol='2', l = '3') select '1' from src limit 1; +insert overwrite table ppr_test3 partition(col='1', ol='1', l = '2') select '2' from src limit 1; +insert overwrite table ppr_test3 partition(col='1', ol='2', l = '1') select '3' from src limit 1; + +select * from ppr_test3 where l = '1'; +select * from ppr_test3 where l = '2'; +select * from ppr_test3 where ol = '1'; +select * from ppr_test3 where ol = '2'; +select * from ppr_test3 where col = '1'; +select * from ppr_test3 where ol = '2' and l = '1'; +select * from ppr_test3 where col='1' and ol = '2' and l = '1'; Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1083890) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -1045,7 +1045,7 @@ * if true - replace files in the partition, otherwise add files to * the partition * @param holdDDLTime if true, force [re]create the partition - * @param inheritTableSpecs if true, on [re]creating the partition, take the + * @param inheritTableSpecs if true, on [re]creating the partition, take the * location/inputformat/outputformat/serde details from table spec * @param tmpDirPath * The temporary directory. @@ -1071,7 +1071,7 @@ Path newPartPath = null; - + if (inheritTableSpecs) { Path partPath = new Path(tbl.getDataLocation().getPath(), Warehouse.makePartPath(partSpec)); @@ -1080,7 +1080,7 @@ } else { newPartPath = oldPartPath; } - + if (replace) { Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf()); } else { @@ -1208,7 +1208,7 @@ */ public Partition createPartition(Table tbl, Map partSpec) throws HiveException { - return createPartition(tbl, partSpec, null, null, null, null, -1, + return createPartition(tbl, partSpec, null, null, null, null, -1, null, null, null, null, null); } @@ -1231,7 +1231,7 @@ * @param serdeParams the serde parameters * @param bucketCols the bucketing columns * @param sortCols sort columns and order - * + * * @return created partition object * @throws HiveException * if table doesn't exist or partition already exists @@ -1310,7 +1310,7 @@ * if this is true and partition doesn't exist then a partition is * created * @param partPath the path where the partition data is located - * @param inheritTableSpecs whether to copy over the table specs for if/of/serde + * @param inheritTableSpecs whether to copy over the table specs for if/of/serde * @return result partition object or null if there is no partition * @throws HiveException */ @@ -1544,7 +1544,37 @@ return partitions; } + /** + * Get a list of Partitions by filter. + * @param tbl The table containing the partitions. + * @param filter A string represent partition predicates. + * @return a list of partitions satisfying the partition predicates. + * @throws HiveException + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + public List getPartitionsByFilter(Table tbl, String filter) + throws HiveException, MetaException, NoSuchObjectException, TException { + + if (!tbl.isPartitioned()) { + throw new HiveException("Partition spec should only be supplied for a " + + "partitioned table"); + } + + List tParts = getMSC().listPartitionsByFilter( + tbl.getDbName(), tbl.getTableName(), filter, (short)-1); + List results = new ArrayList(tParts.size()); + + for (org.apache.hadoop.hive.metastore.api.Partition tPart: tParts) { + Partition part = new Partition(tbl, tPart); + results.add(part); + } + return results; + } + + /** * Get the name of the current database * @return */ Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (revision 1083890) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (working copy) @@ -23,12 +23,16 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; @@ -47,9 +51,14 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.thrift.TException; /** * The transformation step that does partition pruning. @@ -177,59 +186,46 @@ Object[] rowWithPart = new Object[2]; if (tab.isPartitioned()) { - LOG.debug("tabname = " + tab.getTableName() + " is partitioned"); - - for (String partName : Hive.get().getPartitionNames(tab.getDbName(), - tab.getTableName(), (short) -1)) { - // If the "strict" mode is on, we have to provide partition pruner for - // each table. - if ("strict".equalsIgnoreCase(HiveConf.getVar(conf, - HiveConf.ConfVars.HIVEMAPREDMODE))) { - if (!hasColumnExpr(prunerExpr)) { - throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE - .getMsg("for Alias \"" + alias + "\" Table \"" - + tab.getTableName() + "\"")); - } + // If the "strict" mode is on, we have to provide partition pruner for + // each table. + if ("strict".equalsIgnoreCase(HiveConf.getVar(conf, + HiveConf.ConfVars.HIVEMAPREDMODE))) { + if (!hasColumnExpr(prunerExpr)) { + throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE + .getMsg("for Alias \"" + alias + "\" Table \"" + + tab.getTableName() + "\"")); } + } - // Set all the variables here - LinkedHashMap partSpec = Warehouse - .makeSpecFromName(partName); + if (prunerExpr == null) { + // add all partitions corresponding to the table + true_parts.addAll(Hive.get().getPartitions(tab)); + } else { + // remove non-partition columns + ExprNodeDesc compactExpr = prunerExpr.clone(); + compactExpr = compactExpr(compactExpr); + LOG.debug("Filter w/ compacting: " + + ((compactExpr != null) ? compactExpr.getExprString(): "null") + + "; filter w/o compacting: " + + ((prunerExpr != null) ? prunerExpr.getExprString(): "null")); + if (compactExpr == null) { + true_parts.addAll(Hive.get().getPartitions(tab)); + } else if (Utilities.checkJDOPushDown(tab, compactExpr)) { + String filter = compactExpr.getExprString(); + String oldFilter = prunerExpr.getExprString(); - LOG.trace("about to process partition " + partSpec + " for pruning "); - // evaluate the expression tree - if (prunerExpr != null) { - Boolean r = (Boolean) PartExprEvalUtils.evalExprWithPart(prunerExpr, partSpec, - rowObjectInspector); - - LOG.trace("prune result for partition " + partSpec + ": " + r); - if (Boolean.FALSE.equals(r)) { - if (denied_parts.isEmpty()) { - Partition part = Hive.get().getPartition(tab, partSpec, - Boolean.FALSE); - denied_parts.add(part); - } - LOG.trace("pruned partition: " + partSpec); + if (filter.equals(oldFilter)) { + // pruneExpr contains only partition columns + pruneByPushDown(tab, true_parts, filter); } else { - Partition part = Hive.get().getPartition(tab, partSpec, - Boolean.FALSE); - String state = "retained"; - if (Boolean.TRUE.equals(r)) { - true_parts.add(part); - } else { - unkn_parts.add(part); - state = "unknown"; - } - if (LOG.isDebugEnabled()) { - LOG.debug(state + " partition: " + partSpec); - } + // pruneExpr contains non-partition columns + pruneByPushDown(tab, unkn_parts, filter); } } else { - // is there is no parition pruning, all of them are needed - true_parts.add(Hive.get() - .getPartition(tab, partSpec, Boolean.FALSE)); + pruneBySequentialScan(tab, true_parts, unkn_parts, denied_parts, prunerExpr, rowObjectInspector); } } + LOG.debug("tabname = " + tab.getTableName() + " is partitioned"); } else { true_parts.addAll(Hive.get().getPartitions(tab)); } @@ -246,6 +242,120 @@ } /** + * Taking a partition pruning expression, remove the null operands. + * @param expr original partition pruning expression. + * @return partition pruning expression that only contains partition columns. + */ + static private ExprNodeDesc compactExpr(ExprNodeDesc expr) { + if (expr instanceof ExprNodeConstantDesc) { + if (((ExprNodeConstantDesc)expr).getValue() == null) { + return null; + } else { + return expr; + } + } else if (expr instanceof ExprNodeGenericFuncDesc) { + GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF(); + if (udf instanceof GenericUDFOPAnd || + udf instanceof GenericUDFOPOr) { + List children = ((ExprNodeGenericFuncDesc)expr).getChildren(); + ExprNodeDesc left = children.get(0); + children.set(0, compactExpr(left)); + ExprNodeDesc right = children.get(1); + children.set(1, compactExpr(right)); + if (children.get(0) == null && children.get(1) == null) { + return null; + } else if (children.get(0) == null) { + return children.get(1); + } else if (children.get(1) == null) { + return children.get(0); + } + } + return expr; + } + return expr; + } + + /** + * Pruning partition using JDO filtering. + * @param tab the table containing the partitions. + * @param true_parts the resulting partitions. + * @param filter the SQL predicate that involves only partition columns + * @throws HiveException + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + static private void pruneByPushDown(Table tab, Set true_parts, String filter) + throws HiveException, MetaException, NoSuchObjectException, TException { + Hive db = Hive.get(); + List parts = db.getPartitionsByFilter(tab, filter); + true_parts.addAll(parts); + return; + } + + /** + * Pruning partition by getting the partition names first and pruning using Hive expression + * evaluator. + * @param tab the table containing the partitions. + * @param true_parts the resulting partitions if the partition pruning expression only contains + * partition columns. + * @param unkn_parts the resulting partitions if the partition pruning expression that only contains + * non-partition columns. + * @param denied_parts pruned out partitions. + * @param prunerExpr the SQL predicate that involves partition columns. + * @param rowObjectInspector object inspector used by the evaluator + * @throws Exception + */ + static private void pruneBySequentialScan(Table tab, Set true_parts, Set unkn_parts, + Set denied_parts, ExprNodeDesc prunerExpr, StructObjectInspector rowObjectInspector) + throws Exception { + + for (String partName : Hive.get().getPartitionNames(tab.getDbName(), + tab.getTableName(), (short) -1)) { + + // Set all the variables here + LinkedHashMap partSpec = Warehouse + .makeSpecFromName(partName); + + LOG.trace("about to process partition " + partSpec + " for pruning "); + // evaluate the expression tree + if (prunerExpr != null) { + + Boolean r = (Boolean) PartExprEvalUtils.evalExprWithPart(prunerExpr, partSpec, + rowObjectInspector); + + if (Boolean.FALSE.equals(r)) { + if (denied_parts.isEmpty()) { + Partition part = Hive.get().getPartition(tab, partSpec, + Boolean.FALSE); + denied_parts.add(part); + } + LOG.trace("pruned partition: " + partSpec); + } else { + Partition part = Hive.get().getPartition(tab, partSpec, + Boolean.FALSE); + String state = "retained"; + if (Boolean.TRUE.equals(r)) { + true_parts.add(part); + } else { + // r == null means prunerExpr contains null subexpression, + // which was converted from non-partition columns + assert (r == null); + unkn_parts.add(part); + state = "unknown"; + } + if (LOG.isDebugEnabled()) { + LOG.debug(state + " partition: " + partSpec); + } + } + } else { + // is there is no parition pruning, all of them are needed + true_parts.add(Hive.get() + .getPartition(tab, partSpec, Boolean.FALSE)); + } + } + } + /** * Whether the expression contains a column node or not. */ public static boolean hasColumnExpr(ExprNodeDesc desc) { Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1083890) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -93,7 +93,10 @@ import org.apache.hadoop.hive.ql.parse.ErrorMsg; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; +import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; @@ -104,10 +107,15 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.stats.StatsFactory; import org.apache.hadoop.hive.ql.stats.StatsPublisher; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.Serializer; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; @@ -1758,6 +1766,66 @@ } } + /** + * Check if a function can be pushed down to JDO. + * Now only {=, AND, OR} are supported. + * @param func a generic function. + * @return true if this function can be pushed down to JDO filter. + */ + private static boolean supportedJDOFuncs(GenericUDF func) { + return func instanceof GenericUDFOPEqual || + func instanceof GenericUDFOPAnd || + func instanceof GenericUDFOPOr; + } + + /** + * Check if the partition pruning expression can be pushed down to JDO filtering. + * The partition expression contains only partition columns. + * The criteria that an expression can be pushed down are that: + * 1) the expression only contains function specified in supportedJDOFuncs(). + * Now only {=, AND, OR} can be pushed down. + * 2) the partition column type and the constant type have to be String. This is + * restriction by the current JDO filtering implementation. + * @param tab The table that contains the partition columns. + * @param expr the partition pruning expression + * @return true if the partition pruning expression can be pushed down to JDO filtering. + */ + public static boolean checkJDOPushDown(Table tab, ExprNodeDesc expr) { + if (expr instanceof ExprNodeConstantDesc) { + // JDO filter now only support String typed literal -- see Filter.g and ExpressionTree.java + Object value = ((ExprNodeConstantDesc)expr).getValue(); + return (value instanceof String); + } else if (expr instanceof ExprNodeColumnDesc) { + // JDO filter now only support String typed literal -- see Filter.g and ExpressionTree.java + TypeInfo type = expr.getTypeInfo(); + if (type.getTypeName().equals(Constants.STRING_TYPE_NAME)) { + String colName = ((ExprNodeColumnDesc)expr).getColumn(); + for (FieldSchema fs: tab.getPartCols()) { + if (fs.getName().equals(colName)) { + return fs.getType().equals(Constants.STRING_TYPE_NAME); + } + } + assert(false); // cannot find the partition column! + } else { + return false; + } + } else if (expr instanceof ExprNodeGenericFuncDesc) { + ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) expr; + GenericUDF func = funcDesc.getGenericUDF(); + if (!supportedJDOFuncs(func)) { + return false; + } + List children = funcDesc.getChildExprs(); + for (ExprNodeDesc child: children) { + if (!checkJDOPushDown(tab, child)) { + return false; + } + } + return true; + } + return false; + } + private static ThreadLocal> perfKeyMaps = new ThreadLocal>(); /**