diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index b805294..bc81a96 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1508,8 +1508,16 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St if (!success) { ms.rollbackTransaction(); } else if (deleteData && !isExternal) { - boolean ifPurge = envContext != null && - Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + // Data needs deletion. Check if trash may be skipped. + // Trash may be skipped iff: + // 1. deleteData == true, obviously. + // 2. tbl is external. + // 3. Either + // 3.1. User has specified PURGE from the commandline, and if not, + // 3.2. User has set the table to auto-purge. + boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) + || + (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); // Delete the data in the partitions which have other locations deletePartitionData(partPaths, ifPurge); // Delete the data in the table @@ -2552,8 +2560,15 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { if (tbl != null && !isExternal(tbl)) { // Data needs deletion. Check if trash may be skipped. - boolean mustPurge = (envContext != null) - && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + // Trash may be skipped iff: + // 1. deleteData == true, obviously. + // 2. tbl is external. + // 3. Either + // 3.1. User has specified PURGE from the commandline, and if not, + // 3.2. User has set the table to auto-purge. + boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) + || + (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); if (mustPurge) { LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash."); } @@ -2714,8 +2729,15 @@ public DropPartitionsResult drop_partitions_req( ms.rollbackTransaction(); } else if (deleteData && !isExternal(tbl)) { // Data needs deletion. Check if trash may be skipped. - boolean mustPurge = (envContext != null) - && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + // Trash may be skipped iff: + // 1. deleteData == true, obviously. + // 2. tbl is external. + // 3. Either + // 3.1. User has specified PURGE from the commandline, and if not, + // 3.2. User has set the table to auto-purge. + boolean mustPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge"))) + || + (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge"))); LOG.info( mustPurge? "dropPartition() will purge partition-directories directly, skipping trash." : "dropPartition() will move partition-directories to trash-directory."); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index 181cb2a..e0496ce 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -446,7 +446,7 @@ private Table createPartitionedTable(String dbName, String tableName) throws Exc } catch (Exception exception) { fail("Unable to drop and create table " + dbName + "." + tableName - + " because " + StringUtils.stringifyException(exception)); + + " because " + StringUtils.stringifyException(exception)); throw exception; } } @@ -522,6 +522,61 @@ public void testDropPartitionsWithPurge() throws Exception { } } + /** + * Test that tables set up with auto-purge skip trash-directory when tables/partitions are dropped. + * @throws Throwable + */ + public void testAutoPurgeTablesAndPartitions() throws Throwable { + + String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; + String tableName = "table_for_testAutoPurgeTablesAndPartitions"; + try { + + Table table = createPartitionedTable(dbName, tableName); + table.getParameters().put("auto.purge", "true"); + hm.alterTable(tableName, table); + + Map partitionSpec = new ImmutableMap.Builder() + .put("ds", "20141216") + .put("hr", "12") + .build(); + + int trashSizeBeforeDrop = getTrashContents().length; + + hm.createPartition(table, partitionSpec); + + Partition partition = hm.getPartition(table, partitionSpec, false); + assertNotNull("Newly created partition shouldn't be null!", partition); + + hm.dropPartition(dbName, tableName, + partition.getValues(), + true,// Delete data. + false // Purge. + ); + + int trashSizeAfterDrop = getTrashContents().length; + + assertEquals("After dropPartition(noPurge), data should still have skipped trash.", + trashSizeBeforeDrop, trashSizeAfterDrop); + + // Repeat the same check for dropTable. + + trashSizeBeforeDrop = trashSizeAfterDrop; + hm.dropTable(dbName, tableName); + trashSizeAfterDrop = getTrashContents().length; + + assertEquals("After dropTable(noPurge), data should still have skipped trash.", + trashSizeBeforeDrop, trashSizeAfterDrop); + + } + catch(Exception e) { + fail("Unexpected failure: " + StringUtils.stringifyException(e)); + } + finally { + cleanUpTableQuietly(dbName, tableName); + } + } + public void testPartition() throws Throwable { try { String tableName = "table_for_testpartition";