Index: conf/hive-default.xml.template =================================================================== --- conf/hive-default.xml.template (revision 1350025) +++ conf/hive-default.xml.template (working copy) @@ -298,12 +298,6 @@ - hive.metastore.batch.retrieve.table.partition.max - 1000 - Maximum number of table partitions that metastore internally retrieves in one batch. - - - hive.default.fileformat TextFile Default file format for CREATE TABLE statement. Options are TextFile and SequenceFile. Users can explicitly say CREATE TABLE ... STORED AS <TEXTFILE|SEQUENCEFILE> to override @@ -1304,7 +1298,7 @@ set the header will be that borrowed from sequence files, e.g. SEQ- followed by the input and output RC File formats. - + Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1350025) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -606,6 +606,8 @@ IOException { boolean success = false; Database db = null; + List tablePaths = new ArrayList(); + List partitionPaths = new ArrayList(); try { ms.openTransaction(); db = ms.getDatabase(name); @@ -624,6 +626,60 @@ path + " is not writable by " + hiveConf.getUser()); } + + Path databasePath = wh.getDnsPath(wh.getDatabasePath(db)); + + // first drop tables + int tableBatchSize = HiveConf.getIntVar(hiveConf, + ConfVars.METASTORE_BATCH_RETRIEVE_MAX); + + int startIndex = 0; + int endIndex = -1; + // retrieve the tables from the metastore in batches to alleviate memory constraints + while(endIndex < allTables.size() - 1) { + startIndex = endIndex + 1; + endIndex = endIndex + tableBatchSize; + if (endIndex >= allTables.size()) { + endIndex = allTables.size() - 1; + } + + List tables = null; + try { + tables = ms.getTableObjectsByName(name, allTables.subList(startIndex, endIndex)); + } catch (UnknownDBException e) { + throw new MetaException(e.getMessage()); + } + + if (tables != null && !tables.isEmpty()) { + for (Table table : tables) { + + // If the table is not external and it might not be in a subdirectory of the database + // add it's locations to the list of paths to delete + Path tablePath = null; + if (table.getSd().getLocation() != null && !isExternal(table)) { + tablePath = wh.getDnsPath(new Path(table.getSd().getLocation())); + if (!wh.isWritable(tablePath.getParent())) { + throw new MetaException("Database metadata not deleted since table: " + + table.getTableName() + " has a parent location " + tablePath.getParent() + + " which is not writable by " + hiveConf.getUser()); + } + + if (!isSubdirectory(databasePath, tablePath)) { + tablePaths.add(tablePath); + } + } + + // For each partition in each table, drop the partitions and get a list of + // partitions' locations which might need to be deleted + partitionPaths = dropPartitionsAndGetLocations(ms, name, table.getTableName(), + tablePath, table.getPartitionKeys(), deleteData && !isExternal(table)); + + // Drop the table but not its data + drop_table(name, table.getTableName(), false); + } + } + } + if (ms.dropDatabase(name)) { success = ms.commitTransaction(); } @@ -631,7 +687,19 @@ if (!success) { ms.rollbackTransaction(); } else if (deleteData) { - wh.deleteDir(new Path(db.getLocationUri()), true); + // Delete the data in the partitions which have other locations + deletePartitionData(partitionPaths); + // Delete the data in the tables which have other locations + for (Path tablePath : tablePaths) { + deleteTableData(tablePath); + } + // Delete the data in the database + try { + wh.deleteDir(new Path(db.getLocationUri()), true); + } catch (Exception e) { + LOG.error("Failed to delete database directory: " + db.getLocationUri() + + " " + e.getMessage()); + } // it is not a terrible thing even if the data is not deleted } for (MetaStoreEventListener listener : listeners) { @@ -640,6 +708,19 @@ } } + /** + * Returns a BEST GUESS as to whether or not other is a subdirectory of parent. It does not + * take into account any intricacies of the underlying file system, which is assumed to be + * HDFS. This should not return any false positives, but may return false negatives. + * @param parent + * @param other + * @return + */ + private boolean isSubdirectory(Path parent, Path other) { + return other.toString().startsWith(parent.toString().endsWith(Path.SEPARATOR) ? + parent.toString() : parent.toString() + Path.SEPARATOR); + } + public void drop_database(final String dbName, final boolean deleteData, final boolean cascade) throws NoSuchObjectException, InvalidOperationException, MetaException { @@ -907,6 +988,7 @@ boolean success = false; boolean isExternal = false; Path tblPath = null; + List partPaths = null; Table tbl = null; isExternal = false; boolean isIndexTable = false; @@ -958,6 +1040,10 @@ } } + // Drop the partitions and get a list of locations which need to be deleted + partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, + tbl.getPartitionKeys(), deleteData && !isExternal); + if (!ms.dropTable(dbname, name)) { throw new MetaException("Unable to drop table"); } @@ -965,8 +1051,11 @@ } finally { if (!success) { ms.rollbackTransaction(); - } else if (deleteData && (tblPath != null) && !isExternal) { - wh.deleteDir(tblPath, true); + } else if (deleteData && !isExternal) { + // Delete the data in the partitions which have other locations + deletePartitionData(partPaths); + // Delete the data in the table + deleteTableData(tblPath); // ok even if the data is not deleted } for (MetaStoreEventListener listener : listeners) { @@ -975,6 +1064,102 @@ } } + /** + * Deletes the data in a table's location, if it fails logs an error + * + * @param tablePath + */ + private void deleteTableData(Path tablePath) { + if (tablePath != null) { + try { + wh.deleteDir(tablePath, true); + } catch (Exception e) { + LOG.error("Failed to delete table directory: " + tablePath + + " " + e.getMessage()); + } + } + } + + /** + * Give a list of partitions' locations, tries to delete each one + * and for each that fails logs an error. + * + * @param partPaths + */ + private void deletePartitionData(List partPaths) { + if (partPaths != null && !partPaths.isEmpty()) { + for (Path partPath : partPaths) { + try { + wh.deleteDir(partPath, true); + } catch (Exception e) { + LOG.error("Failed to delete partition directory: " + partPath + + " " + e.getMessage()); + } + } + } + } + + /** + * Retrieves the partitions specified by partitionKeys. If checkLocation, for locations of + * partitions which may not be subdirectories of tablePath checks to make the locations are + * writable. + * + * Drops the metadata for each partition. + * + * Provides a list of locations of partitions which may not be subdirectories of tablePath. + * + * @param ms + * @param dbName + * @param tableName + * @param tablePath + * @param partitionKeys + * @param checkLocation + * @return + * @throws MetaException + * @throws IOException + */ + private List dropPartitionsAndGetLocations(RawStore ms, String dbName, + String tableName, Path tablePath, List partitionKeys, boolean checkLocation) + throws MetaException, IOException { + + int partitionBatchSize = HiveConf.getIntVar(hiveConf, + ConfVars.METASTORE_BATCH_RETRIEVE_MAX); + Path tableDnsPath = null; + if (tablePath != null) { + tableDnsPath = wh.getDnsPath(tablePath); + } + List partPaths = new ArrayList(); + + // call dropPartition on each of the table's partitions to follow the + // procedure for cleanly dropping partitions. + while(true) { + List partsToDelete = ms.getPartitions(dbName, tableName, partitionBatchSize); + if (partsToDelete == null || partsToDelete.isEmpty()) { + break; + } + for (Partition part : partsToDelete) { + if (checkLocation && part.getSd() != null && + part.getSd().getLocation() != null) { + + Path partPath = wh.getDnsPath(new Path(part.getSd().getLocation())); + if (tableDnsPath == null || + (partPath != null && !isSubdirectory(tableDnsPath, partPath))){ + if (!wh.isWritable(partPath.getParent())) { + throw new MetaException("Table metadata not deleted since the partition " + + Warehouse.makePartName(partitionKeys, part.getValues()) + + " has parent location " + partPath.getParent() + " which is not writable " + + "by " + hiveConf.getUser()); + } + partPaths.add(partPath); + } + } + ms.dropPartition(dbName, tableName, part.getValues()); + } + } + + return partPaths; + } + public void drop_table(final String dbname, final String name, final boolean deleteData) throws NoSuchObjectException, MetaException { startTableFunction("drop_table", dbname, name); @@ -2149,6 +2334,7 @@ boolean success = false; Path tblPath = null; + List partPaths = null; try { ms.openTransaction(); @@ -2175,6 +2361,11 @@ hiveConf.getUser()); } } + + // Drop the partitions and get a list of partition locations which need to be deleted + partPaths = dropPartitionsAndGetLocations(ms, dbName, idxTblName, tblPath, + tbl.getPartitionKeys(), deleteData); + if (!ms.dropTable(dbName, idxTblName)) { throw new MetaException("Unable to drop underlying data table " + idxTblName + " for index " + idxTblName); @@ -2186,7 +2377,8 @@ ms.rollbackTransaction(); return false; } else if (deleteData && tblPath != null) { - wh.deleteDir(tblPath, true); + deletePartitionData(partPaths); + deleteTableData(tblPath); // ok even if the data is not deleted } } Index: metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (revision 1350025) +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (working copy) @@ -456,11 +456,6 @@ try { openTransaction(); - // first drop tables - for (String tableName : getAllTables(dbname)) { - dropTable(dbname, tableName); - } - // then drop the database MDatabase db = getMDatabase(dbname); pm.retrieve(db); @@ -681,7 +676,7 @@ MTable tbl = getMTable(dbName, tableName); pm.retrieve(tbl); if (tbl != null) { - // first remove all the partitions + // first remove all the grants List tabGrants = listAllTableGrants(dbName, tableName); if (tabGrants != null && tabGrants.size() > 0) { pm.deletePersistentAll(tabGrants); @@ -703,21 +698,6 @@ pm.deletePersistentAll(partColGrants); } - int partitionBatchSize = HiveConf.getIntVar(getConf(), - ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX); - - // call dropPartition on each of the table's partitions to follow the - // procedure for cleanly dropping partitions. - while(true) { - List partsToDelete = listMPartitions(dbName, tableName, partitionBatchSize); - if (partsToDelete == null || partsToDelete.isEmpty()) { - break; - } - for (MPartition mpart : partsToDelete) { - dropPartitionCommon(mpart); - } - } - preDropStorageDescriptor(tbl.getSd()); // then remove the table pm.deletePersistentAll(tbl); Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1350025) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -122,7 +122,6 @@ HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS, HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS, - HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX, HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS, }; @@ -291,8 +290,6 @@ METASTORE_IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus"), METASTORE_PLUGIN_REGISTRY_BUNDLE_CHECK("datanucleus.plugin.pluginRegistryBundleCheck", "LOG"), METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300), - METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX( - "hive.metastore.batch.retrieve.table.partition.max", 1000), METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", ""), METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", ""), // should we do checks against the storage (usually hdfs) for operations like drop_partition Index: ql/src/test/results/clientpositive/drop_table_removes_partition_dirs.q.out =================================================================== --- ql/src/test/results/clientpositive/drop_table_removes_partition_dirs.q.out (revision 0) +++ ql/src/test/results/clientpositive/drop_table_removes_partition_dirs.q.out (revision 0) @@ -0,0 +1,51 @@ +PREHOOK: query: -- This test verifies that if a partition exists outside the table's current location when the +-- table is dropped the partition's location is dropped as well. + +CREATE TABLE drop_table_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- This test verifies that if a partition exists outside the table's current location when the +-- table is dropped the partition's location is dropped as well. + +CREATE TABLE drop_table_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@drop_table_removes_partition_dirs_table +PREHOOK: query: ALTER TABLE drop_table_removes_partition_dirs_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@drop_table_removes_partition_dirs_table +POSTHOOK: query: ALTER TABLE drop_table_removes_partition_dirs_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@drop_table_removes_partition_dirs_table +POSTHOOK: Output: default@drop_table_removes_partition_dirs_table@part=1 +PREHOOK: query: INSERT OVERWRITE TABLE drop_table_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@drop_table_removes_partition_dirs_table@part=1 +POSTHOOK: query: INSERT OVERWRITE TABLE drop_table_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@drop_table_removes_partition_dirs_table@part=1 +POSTHOOK: Lineage: drop_table_removes_partition_dirs_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: drop_table_removes_partition_dirs_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +Found 1 items +#### A masked pattern was here #### +PREHOOK: query: DROP TABLE drop_table_removes_partition_dirs_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@drop_table_removes_partition_dirs_table +PREHOOK: Output: default@drop_table_removes_partition_dirs_table +POSTHOOK: query: DROP TABLE drop_table_removes_partition_dirs_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@drop_table_removes_partition_dirs_table +POSTHOOK: Output: default@drop_table_removes_partition_dirs_table +POSTHOOK: Lineage: drop_table_removes_partition_dirs_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: drop_table_removes_partition_dirs_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +#### A masked pattern was here #### Index: ql/src/test/results/clientpositive/drop_database_removes_partition_dirs.q.out =================================================================== --- ql/src/test/results/clientpositive/drop_database_removes_partition_dirs.q.out (revision 0) +++ ql/src/test/results/clientpositive/drop_database_removes_partition_dirs.q.out (revision 0) @@ -0,0 +1,61 @@ +PREHOOK: query: -- This test verifies that if a partition exists outside a table's current location when the +-- database is dropped the partition's location is dropped as well. + +CREATE DATABASE drop_database_removes_partition_dirs_database +PREHOOK: type: CREATEDATABASE +POSTHOOK: query: -- This test verifies that if a partition exists outside a table's current location when the +-- database is dropped the partition's location is dropped as well. + +CREATE DATABASE drop_database_removes_partition_dirs_database +POSTHOOK: type: CREATEDATABASE +PREHOOK: query: USE drop_database_removes_partition_dirs_database +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: USE drop_database_removes_partition_dirs_database +POSTHOOK: type: SWITCHDATABASE +PREHOOK: query: CREATE TABLE drop_database_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +POSTHOOK: query: CREATE TABLE drop_database_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table +PREHOOK: query: ALTER TABLE drop_database_removes_partition_dirs_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table +POSTHOOK: query: ALTER TABLE drop_database_removes_partition_dirs_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table +POSTHOOK: Output: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table@part=1 +PREHOOK: query: INSERT OVERWRITE TABLE drop_database_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM default.src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table@part=1 +POSTHOOK: query: INSERT OVERWRITE TABLE drop_database_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM default.src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: drop_database_removes_partition_dirs_database@drop_database_removes_partition_dirs_table@part=1 +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +Found 1 items +#### A masked pattern was here #### +PREHOOK: query: USE default +PREHOOK: type: SWITCHDATABASE +POSTHOOK: query: USE default +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: DROP DATABASE drop_database_removes_partition_dirs_database CASCADE +PREHOOK: type: DROPDATABASE +POSTHOOK: query: DROP DATABASE drop_database_removes_partition_dirs_database CASCADE +POSTHOOK: type: DROPDATABASE +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: drop_database_removes_partition_dirs_table PARTITION(part=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +#### A masked pattern was here #### Index: ql/src/test/results/clientpositive/drop_index_removes_partition_dirs.q.out =================================================================== --- ql/src/test/results/clientpositive/drop_index_removes_partition_dirs.q.out (revision 0) +++ ql/src/test/results/clientpositive/drop_index_removes_partition_dirs.q.out (revision 0) @@ -0,0 +1,42 @@ +PREHOOK: query: -- This test verifies that if a partition exists outside an index table's current location when the +-- index is dropped the partition's location is dropped as well. + +CREATE TABLE drop_index_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- This test verifies that if a partition exists outside an index table's current location when the +-- index is dropped the partition's location is dropped as well. + +CREATE TABLE drop_index_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@drop_index_removes_partition_dirs_table +PREHOOK: query: CREATE INDEX drop_index_removes_partition_dirs_index ON +TABLE drop_index_removes_partition_dirs_table(key) AS 'compact' WITH DEFERRED REBUILD +IN TABLE drop_index_removes_partition_dirs_index_table +PREHOOK: type: CREATEINDEX +POSTHOOK: query: CREATE INDEX drop_index_removes_partition_dirs_index ON +TABLE drop_index_removes_partition_dirs_table(key) AS 'compact' WITH DEFERRED REBUILD +IN TABLE drop_index_removes_partition_dirs_index_table +POSTHOOK: type: CREATEINDEX +POSTHOOK: Output: default@drop_index_removes_partition_dirs_index_table +PREHOOK: query: ALTER TABLE drop_index_removes_partition_dirs_index_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@drop_index_removes_partition_dirs_index_table +POSTHOOK: query: ALTER TABLE drop_index_removes_partition_dirs_index_table ADD PARTITION (part = '1') +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@drop_index_removes_partition_dirs_index_table +POSTHOOK: Output: default@drop_index_removes_partition_dirs_index_table@part=1 +Found 1 items +#### A masked pattern was here #### +PREHOOK: query: DROP INDEX drop_index_removes_partition_dirs_index ON drop_index_removes_partition_dirs_table +PREHOOK: type: DROPINDEX +POSTHOOK: query: DROP INDEX drop_index_removes_partition_dirs_index ON drop_index_removes_partition_dirs_table +POSTHOOK: type: DROPINDEX +#### A masked pattern was here #### Index: ql/src/test/queries/clientpositive/drop_database_removes_partition_dirs.q =================================================================== --- ql/src/test/queries/clientpositive/drop_database_removes_partition_dirs.q (revision 0) +++ ql/src/test/queries/clientpositive/drop_database_removes_partition_dirs.q (revision 0) @@ -0,0 +1,27 @@ +-- This test verifies that if a partition exists outside a table's current location when the +-- database is dropped the partition's location is dropped as well. + +CREATE DATABASE drop_database_removes_partition_dirs_database; + +USE drop_database_removes_partition_dirs_database; + +CREATE TABLE drop_database_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +LOCATION 'file:${system:test.tmp.dir}/drop_database_removes_partition_dirs_table'; + +ALTER TABLE drop_database_removes_partition_dirs_table ADD PARTITION (part = '1') +LOCATION 'file:${system:test.tmp.dir}/drop_database_removes_partition_dirs_table2/part=1'; + +INSERT OVERWRITE TABLE drop_database_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM default.src; + +dfs -ls ${system:test.tmp.dir}/drop_database_removes_partition_dirs_table2; + +USE default; + +DROP DATABASE drop_database_removes_partition_dirs_database CASCADE; + +dfs -ls ${system:test.tmp.dir}/drop_database_removes_partition_dirs_table2; + +dfs -rmr ${system:test.tmp.dir}/drop_database_removes_partition_dirs_table2; \ No newline at end of file Index: ql/src/test/queries/clientpositive/drop_table_removes_partition_dirs.q =================================================================== --- ql/src/test/queries/clientpositive/drop_table_removes_partition_dirs.q (revision 0) +++ ql/src/test/queries/clientpositive/drop_table_removes_partition_dirs.q (revision 0) @@ -0,0 +1,21 @@ +-- This test verifies that if a partition exists outside the table's current location when the +-- table is dropped the partition's location is dropped as well. + +CREATE TABLE drop_table_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +LOCATION 'file:${system:test.tmp.dir}/drop_table_removes_partition_dirs_table'; + +ALTER TABLE drop_table_removes_partition_dirs_table ADD PARTITION (part = '1') +LOCATION 'file:${system:test.tmp.dir}/drop_table_removes_partition_dirs_table2/part=1'; + +INSERT OVERWRITE TABLE drop_table_removes_partition_dirs_table PARTITION (part = '1') +SELECT * FROM src; + +dfs -ls ${system:test.tmp.dir}/drop_table_removes_partition_dirs_table2; + +DROP TABLE drop_table_removes_partition_dirs_table; + +dfs -ls ${system:test.tmp.dir}/drop_table_removes_partition_dirs_table2; + +dfs -rmr ${system:test.tmp.dir}/drop_table_removes_partition_dirs_table2; \ No newline at end of file Index: ql/src/test/queries/clientpositive/drop_index_removes_partition_dirs.q =================================================================== --- ql/src/test/queries/clientpositive/drop_index_removes_partition_dirs.q (revision 0) +++ ql/src/test/queries/clientpositive/drop_index_removes_partition_dirs.q (revision 0) @@ -0,0 +1,22 @@ +-- This test verifies that if a partition exists outside an index table's current location when the +-- index is dropped the partition's location is dropped as well. + +CREATE TABLE drop_index_removes_partition_dirs_table (key STRING, value STRING) +PARTITIONED BY (part STRING) +STORED AS RCFILE +LOCATION 'file:${system:test.tmp.dir}/drop_database_removes_partition_dirs_table'; + +CREATE INDEX drop_index_removes_partition_dirs_index ON +TABLE drop_index_removes_partition_dirs_table(key) AS 'compact' WITH DEFERRED REBUILD +IN TABLE drop_index_removes_partition_dirs_index_table; + +ALTER TABLE drop_index_removes_partition_dirs_index_table ADD PARTITION (part = '1') +LOCATION 'file:${system:test.tmp.dir}/drop_index_removes_partition_dirs_index_table2/part=1'; + +dfs -ls ${system:test.tmp.dir}/drop_index_removes_partition_dirs_index_table2; + +DROP INDEX drop_index_removes_partition_dirs_index ON drop_index_removes_partition_dirs_table; + +dfs -ls ${system:test.tmp.dir}/drop_index_removes_partition_dirs_index_table2; + +dfs -rmr ${system:test.tmp.dir}/drop_index_removes_partition_dirs_index_table2; \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1350025) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -3050,7 +3050,7 @@ } int partitionBatchSize = HiveConf.getIntVar(conf, - ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX); + ConfVars.METASTORE_BATCH_RETRIEVE_MAX); // We should check that all the partitions of the table can be dropped if (tbl != null && tbl.isPartitioned()) {