Index: ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out =================================================================== --- ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out (revision 0) +++ ql/src/test/results/clientnegative/drop_partition_filter_failure2.q.out (revision 1354154) @@ -0,0 +1,34 @@ +PREHOOK: query: create table ptestfilter (a string, b int) partitioned by (c string, d int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ptestfilter (a string, b int) partitioned by (c string, d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ptestfilter +PREHOOK: query: describe ptestfilter +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe ptestfilter +POSTHOOK: type: DESCTABLE +a string +b int +c string +d int +PREHOOK: query: alter table ptestfilter add partition (c='US', d=1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c='US', d=1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=US/d=1 +PREHOOK: query: alter table ptestfilter add partition (c='US', d=2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c='US', d=2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=US/d=2 +PREHOOK: query: show partitions ptestfilter +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions ptestfilter +POSTHOOK: type: SHOWPARTITIONS +c=US/d=1 +c=US/d=2 +FAILED: Error in semantic analysis: Drop partitions for a non string partition columns is not allowed using non-equality Index: ql/src/test/results/clientpositive/drop_partitions_filter2.q.out =================================================================== --- ql/src/test/results/clientpositive/drop_partitions_filter2.q.out (revision 0) +++ ql/src/test/results/clientpositive/drop_partitions_filter2.q.out (revision 1354154) @@ -0,0 +1,107 @@ +PREHOOK: query: create table ptestfilter (a string, b int) partitioned by (c int, d int) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table ptestfilter (a string, b int) partitioned by (c int, d int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@ptestfilter +PREHOOK: query: describe ptestfilter +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe ptestfilter +POSTHOOK: type: DESCTABLE +a string +b int +c int +d int +PREHOOK: query: alter table ptestfilter add partition (c=1, d=1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c=1, d=1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=1/d=1 +PREHOOK: query: alter table ptestfilter add partition (c=1, d=2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c=1, d=2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=1/d=2 +PREHOOK: query: alter table ptestFilter add partition (c=2, d=1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestFilter add partition (c=2, d=1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=2/d=1 +PREHOOK: query: alter table ptestfilter add partition (c=2, d=2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c=2, d=2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=2/d=2 +PREHOOK: query: alter table ptestfilter add partition (c=3, d=1) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c=3, d=1) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=3/d=1 +PREHOOK: query: alter table ptestfilter add partition (c=3, d=2) +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@ptestfilter +POSTHOOK: query: alter table ptestfilter add partition (c=3, d=2) +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=3/d=2 +PREHOOK: query: show partitions ptestfilter +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions ptestfilter +POSTHOOK: type: SHOWPARTITIONS +c=1/d=1 +c=1/d=2 +c=2/d=1 +c=2/d=2 +c=3/d=1 +c=3/d=2 +PREHOOK: query: alter table ptestfilter drop partition (c=1, d=1) +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@ptestfilter +PREHOOK: Output: default@ptestfilter@c=1/d=1 +POSTHOOK: query: alter table ptestfilter drop partition (c=1, d=1) +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=1/d=1 +PREHOOK: query: show partitions ptestfilter +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions ptestfilter +POSTHOOK: type: SHOWPARTITIONS +c=1/d=2 +c=2/d=1 +c=2/d=2 +c=3/d=1 +c=3/d=2 +PREHOOK: query: alter table ptestfilter drop partition (c=2) +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@ptestfilter +PREHOOK: Output: default@ptestfilter@c=2/d=1 +PREHOOK: Output: default@ptestfilter@c=2/d=2 +POSTHOOK: query: alter table ptestfilter drop partition (c=2) +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter@c=2/d=1 +POSTHOOK: Output: default@ptestfilter@c=2/d=2 +PREHOOK: query: show partitions ptestfilter +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions ptestfilter +POSTHOOK: type: SHOWPARTITIONS +c=1/d=2 +c=3/d=1 +c=3/d=2 +PREHOOK: query: drop table ptestfilter +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ptestfilter +PREHOOK: Output: default@ptestfilter +POSTHOOK: query: drop table ptestfilter +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ptestfilter +POSTHOOK: Output: default@ptestfilter Index: ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q =================================================================== --- ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q (revision 0) +++ ql/src/test/queries/clientnegative/drop_partition_filter_failure2.q (revision 1354154) @@ -0,0 +1,11 @@ +create table ptestfilter (a string, b int) partitioned by (c string, d int); +describe ptestfilter; + +alter table ptestfilter add partition (c='US', d=1); +alter table ptestfilter add partition (c='US', d=2); +show partitions ptestfilter; + +alter table ptestfilter drop partition (c='US', d<'2'); + + + Index: ql/src/test/queries/clientpositive/drop_partitions_filter2.q =================================================================== --- ql/src/test/queries/clientpositive/drop_partitions_filter2.q (revision 0) +++ ql/src/test/queries/clientpositive/drop_partitions_filter2.q (revision 1354154) @@ -0,0 +1,20 @@ +create table ptestfilter (a string, b int) partitioned by (c int, d int); +describe ptestfilter; + +alter table ptestfilter add partition (c=1, d=1); +alter table ptestfilter add partition (c=1, d=2); +alter table ptestFilter add partition (c=2, d=1); +alter table ptestfilter add partition (c=2, d=2); +alter table ptestfilter add partition (c=3, d=1); +alter table ptestfilter add partition (c=3, d=2); +show partitions ptestfilter; + +alter table ptestfilter drop partition (c=1, d=1); +show partitions ptestfilter; + +alter table ptestfilter drop partition (c=2); +show partitions ptestfilter; + +drop table ptestfilter; + + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1354153) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1354154) @@ -3002,11 +3002,19 @@ List partsToDelete = new ArrayList(); for (PartitionSpec partSpec : dropTbl.getPartSpecs()) { List partitions = null; - try { - partitions = db.getPartitionsByFilter(tbl, partSpec.toString()); - } catch (Exception e) { - throw new HiveException(e); + // getPartitionsByFilter only works for string columns. + // Till that is fixed, only equality will work for non-string columns. + if (dropTbl.isStringPartitionColumns()) { + try { + partitions = db.getPartitionsByFilter(tbl, partSpec.toString()); + } catch (Exception e) { + throw new HiveException(e); + } } + else { + partitions = db.getPartitions(tbl, partSpec.getPartSpecWithoutOperator()); + } + // this is to prevent dropping archived partition which is archived in a // different level the drop command specified. int partPrefixToDrop = 0; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (revision 1354153) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (revision 1354154) @@ -34,6 +34,9 @@ ArrayList partSpecs; boolean expectView; boolean ifExists; + boolean stringPartitionColumns; // This is due to JDO not working very well with + // non-string partition columns. + // We need a different codepath for them public DropTableDesc() { } @@ -41,14 +44,17 @@ /** * @param tableName */ - public DropTableDesc(String tableName, boolean expectView, boolean ifExists) { + public DropTableDesc(String tableName, boolean expectView, + boolean ifExists, boolean stringPartitionColumns) { this.tableName = tableName; partSpecs = null; this.expectView = expectView; this.ifExists = ifExists; + this.stringPartitionColumns = stringPartitionColumns; } - public DropTableDesc(String tableName, List partSpecs, boolean expectView) { + public DropTableDesc(String tableName, List partSpecs, + boolean expectView, boolean stringPartitionColumns) { this.tableName = tableName; this.partSpecs = new ArrayList(partSpecs.size()); @@ -56,6 +62,7 @@ this.partSpecs.add(partSpecs.get(i)); } this.expectView = expectView; + this.stringPartitionColumns = stringPartitionColumns; } /** @@ -118,4 +125,12 @@ public void setIfExists(boolean ifExists) { this.ifExists = ifExists; } + + public boolean isStringPartitionColumns() { + return stringPartitionColumns; + } + + public void setStringPartitionColumns(boolean stringPartitionColumns) { + this.stringPartitionColumns = stringPartitionColumns; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java (revision 1354153) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionSpec.java (revision 1354154) @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; @@ -62,7 +64,7 @@ } } - private Map partSpec; + private final Map partSpec; public PartitionSpec() { this.partSpec = new LinkedHashMap(); @@ -102,4 +104,29 @@ } return filterString.toString(); } + + // getParitionsByFilter only works for string columns due to a JDO limitation. + // The operator is only useful if it can be passed as a filter to the metastore. + // For compatibility with other non-string partition columns, this function + // returns the key, value mapping assuming that the operator is equality. + public Map getPartSpecWithoutOperator() { + Map partSpec = new HashMap(); + for (Map.Entry entry: this.partSpec.entrySet()) { + partSpec.put(entry.getKey(), entry.getValue().getValue()); + } + + return partSpec; + } + + // Again, for the same reason as the above function - getPartSpecWithoutOperator + public boolean isNonEqualityOperator() { + Iterator iter = partSpec.values().iterator(); + while (iter.hasNext()) { + PredicateSpec predSpec = iter.next(); + if (!predSpec.operator.equals("=")) { + return true; + } + } + return false; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (revision 1354153) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (revision 1354154) @@ -193,6 +193,8 @@ PARTITION_COLUMN_NON_PRIMITIVE("Partition column must be of primitive type."), INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS( "Dynamic partitions do not support IF NOT EXISTS. Specified partitions with value :"), + DROP_PARTITION_NON_STRING_PARTCOLS_NONEQUALITY("Drop partitions for a non string partition " + + "columns is not allowed using non-equality") ; private String mesg; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1354153) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1354154) @@ -707,7 +707,7 @@ } DropTableDesc dropTblDesc = new DropTableDesc( - tableName, expectView, ifExists); + tableName, expectView, ifExists, true); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf)); } @@ -1769,11 +1769,10 @@ String tblName = getUnescapedName((ASTNode)ast.getChild(0)); // get table metadata List partSpecs = getFullPartitionSpecs(ast); - DropTableDesc dropTblDesc = - new DropTableDesc(tblName, partSpecs, expectView); + Table tab = null; try { - Table tab = db.getTable(db.getCurrentDatabase(), tblName, false); + tab = db.getTable(db.getCurrentDatabase(), tblName, false); if (tab != null) { inputs.add(new ReadEntity(tab)); } @@ -1781,15 +1780,39 @@ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName)); } + // Find out if all partition columns are strings. This is needed for JDO + boolean stringPartitionColumns = true; + List partCols = tab.getPartCols(); + + for (FieldSchema partCol : partCols) { + if (!partCol.getType().toLowerCase().equals("string")) { + stringPartitionColumns = false; + break; + } + } + + // Only equality is supported for non-string partition columns + if (!stringPartitionColumns) { + for (PartitionSpec partSpec : partSpecs) { + if (partSpec.isNonEqualityOperator()) { + throw new SemanticException( + ErrorMsg.DROP_PARTITION_NON_STRING_PARTCOLS_NONEQUALITY.getMsg()); + } + } + } + if (partSpecs != null) { boolean ifExists = (ast.getFirstChildWithType(TOK_IFEXISTS) != null); // we want to signal an error if the partition doesn't exist and we're // configured not to fail silently boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); - addTableDropPartsOutputs(tblName, partSpecs, throwException); + addTableDropPartsOutputs(tblName, partSpecs, throwException, stringPartitionColumns); } + DropTableDesc dropTblDesc = + new DropTableDesc(tblName, partSpecs, expectView, stringPartitionColumns); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf)); } @@ -2197,7 +2220,7 @@ * throwIfNonExistent is true, otherwise ignore it. */ private void addTableDropPartsOutputs(String tblName, List partSpecs, - boolean throwIfNonExistent) + boolean throwIfNonExistent, boolean stringPartitionColumns) throws SemanticException { Table tab; try { @@ -2211,11 +2234,21 @@ for (i = partSpecs.iterator(), index = 1; i.hasNext(); ++index) { PartitionSpec partSpec = i.next(); List parts = null; - try { - parts = db.getPartitionsByFilter(tab, partSpec.toString()); - } catch (Exception e) { - throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e); + if (stringPartitionColumns) { + try { + parts = db.getPartitionsByFilter(tab, partSpec.toString()); + } catch (Exception e) { + throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e); + } } + else { + try { + parts = db.getPartitions(tab, partSpec.getPartSpecWithoutOperator()); + } catch (Exception e) { + throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString()), e); + } + } + if (parts.isEmpty()) { if(throwIfNonExistent) { throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partSpec.toString())); Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /hive/trunk:r1344801