diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index d1558876f1..9cf8367cec 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -657,12 +657,20 @@ public boolean generateSqlFilterForPushdown( public boolean generateSqlFilterForPushdown(Table table, ExpressionTree tree, String defaultPartitionName, SqlFilterForPushdown result) throws MetaException { + result.table = table; + return generateSqlFilterForPushdown(table.getCatName(), table.getDbName(), table.getTableName(), + table.getPartitionKeys(), tree, defaultPartitionName, result); + } + + public boolean generateSqlFilterForPushdown(String catName, String dbName, String tableName, + List partitionKeys, ExpressionTree tree, String defaultPartitionName, + SqlFilterForPushdown result) throws MetaException { // Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround. boolean dbHasJoinCastBug = DatabaseProduct.hasJoinOperationOrderBug(dbType); - result.table = table; - result.filter = PartitionFilterGenerator.generateSqlFilter(table, tree, result.params, - result.joins, dbHasJoinCastBug, ((defaultPartitionName == null) ? defaultPartName : defaultPartitionName), - dbType, schema); + result.filter = PartitionFilterGenerator.generateSqlFilter(catName, dbName, tableName, + partitionKeys, tree, result.params, result.joins, dbHasJoinCastBug, + ((defaultPartitionName == null) ? defaultPartName : defaultPartitionName), + dbType, schema); return result.filter != null; } @@ -998,7 +1006,10 @@ private static String trimCommaList(StringBuilder sb) { } private static class PartitionFilterGenerator extends TreeVisitor { - private final Table table; + private String catName; + private String dbName; + private String tableName; + private List partitionKeys; private final FilterBuilder filterBuffer; private final List params; private final List joins; @@ -1007,9 +1018,13 @@ private static String trimCommaList(StringBuilder sb) { private final DatabaseProduct dbType; private final String PARTITION_KEY_VALS, PARTITIONS, DBS, TBLS; - private PartitionFilterGenerator(Table table, List params, List joins, + private PartitionFilterGenerator(String catName, String dbName, String tableName, + List partitionKeys, List params, List joins, boolean dbHasJoinCastBug, String defaultPartName, DatabaseProduct dbType, String schema) { - this.table = table; + this.catName = catName; + this.dbName = dbName; + this.tableName = tableName; + this.partitionKeys = (partitionKeys == null) ? Collections.emptyList() : partitionKeys; this.params = params; this.joins = joins; this.dbHasJoinCastBug = dbHasJoinCastBug; @@ -1024,15 +1039,18 @@ private PartitionFilterGenerator(Table table, List params, List /** * Generate the ANSI SQL92 filter for the given expression tree - * @param table the table being queried + * @param catName catalog name + * @param dbName db name + * @param tableName table name + * @param partitionKeys partition keys * @param params the ordered parameters for the resulting expression * @param joins the joins necessary for the resulting expression * @return the string representation of the expression tree */ - private static String generateSqlFilter(Table table, ExpressionTree tree, List params, + private static String generateSqlFilter(String catName, String dbName, String tableName, + List partitionKeys, ExpressionTree tree, List params, List joins, boolean dbHasJoinCastBug, String defaultPartName, DatabaseProduct dbType, String schema) throws MetaException { - assert table != null; if (tree == null) { // consistent with other APIs like makeExpressionTree, null is returned to indicate that // the filter could not pushed down due to parsing issue etc @@ -1042,7 +1060,8 @@ private static String generateSqlFilter(Table table, ExpressionTree tree, List partitionKeys = convertToFieldSchemas(mTable.getPartitionKeys()); result.addAll(new GetListHelper(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List getSqlResult(GetHelper> ctx) throws MetaException { // If we have some sort of expression tree, try SQL filter pushdown. if (exprTree != null) { SqlFilterForPushdown filter = new SqlFilterForPushdown(); - if (directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, defaultPartitionName, filter)) { + if (directSql.generateSqlFilterForPushdown(catName, dbName, tblName, partitionKeys, + exprTree, defaultPartitionName, filter)) { return directSql.getPartitionsViaSqlFilter(filter, null); } } @@ -3366,7 +3369,7 @@ protected boolean getPartitionsByExprInternal(String catName, String dbName, Str } return result; } - }.run(true)); + }.run(false)); return hasUnknownPartitions.get(); } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java index 9834883f00..97041a3c95 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.Stack; @@ -28,6 +29,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.ColumnType; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -430,16 +432,27 @@ public boolean canJdoUseStringsWithIntegral() { */ public int getPartColIndexForFilter( Table table, FilterBuilder filterBuilder) throws MetaException { + return getPartColIndexForFilter(table.getPartitionKeys(), filterBuilder); + } + + /** + * Get partition column index in the table partition column list that + * corresponds to the key that is being filtered on by this tree node. + * @param partitionKeys list of partition keys. + * @param filterBuilder filter builder used to report error, if any. + * @return The index. + */ + public int getPartColIndexForFilter( + List partitionKeys, FilterBuilder filterBuilder) throws MetaException { + assert (partitionKeys.size() > 0); int partitionColumnIndex; - assert (table.getPartitionKeys().size() > 0); - for (partitionColumnIndex = 0; partitionColumnIndex < table.getPartitionKeys().size(); - ++partitionColumnIndex) { - if (table.getPartitionKeys().get(partitionColumnIndex).getName(). - equalsIgnoreCase(keyName)) { + for (partitionColumnIndex = 0; partitionColumnIndex < partitionKeys.size(); + ++partitionColumnIndex) { + if (partitionKeys.get(partitionColumnIndex).getName().equalsIgnoreCase(keyName)) { break; } } - if( partitionColumnIndex == table.getPartitionKeys().size()) { + if( partitionColumnIndex == partitionKeys.size()) { filterBuilder.setError("Specified key <" + keyName + "> is not a partitioning key for the table"); return -1;