diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 2547da7..b805294 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2551,15 +2551,24 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na ms.rollbackTransaction(); } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { if (tbl != null && !isExternal(tbl)) { + // Data needs deletion. Check if trash may be skipped. + boolean mustPurge = (envContext != null) + && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + if (mustPurge) { + LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash."); + } + else { + LOG.info("dropPartition() will move " + partPath + " to trash-directory."); + } // Archived partitions have har:/to_har_file as their location. // The original directory was saved in params if (isArchived) { assert (archiveParentDir != null); - wh.deleteDir(archiveParentDir, true); + wh.deleteDir(archiveParentDir, true, mustPurge); } else { assert (partPath != null); - wh.deleteDir(partPath, true); - deleteParentRecursive(partPath.getParent(), part_vals.size() - 1); + wh.deleteDir(partPath, true, mustPurge); + deleteParentRecursive(partPath.getParent(), part_vals.size() - 1, mustPurge); } // ok even if the data is not deleted } @@ -2574,10 +2583,10 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na return true; } - private void deleteParentRecursive(Path parent, int depth) throws IOException, MetaException { + private void deleteParentRecursive(Path parent, int depth, boolean mustPurge) throws IOException, MetaException { if (depth > 0 && parent != null && wh.isWritable(parent) && wh.isEmpty(parent)) { - wh.deleteDir(parent, true); - deleteParentRecursive(parent.getParent(), depth - 1); + wh.deleteDir(parent, true, mustPurge); + deleteParentRecursive(parent.getParent(), depth - 1, mustPurge); } } @@ -2704,15 +2713,21 @@ public DropPartitionsResult drop_partitions_req( if (!success) { ms.rollbackTransaction(); } else if (deleteData && !isExternal(tbl)) { + // Data needs deletion. Check if trash may be skipped. + boolean mustPurge = (envContext != null) + && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + LOG.info( mustPurge? + "dropPartition() will purge partition-directories directly, skipping trash." + : "dropPartition() will move partition-directories to trash-directory."); // Archived partitions have har:/to_har_file as their location. // The original directory was saved in params for (Path path : archToDelete) { - wh.deleteDir(path, true); + wh.deleteDir(path, true, mustPurge); } for (PathAndPartValSize p : dirsToDelete) { - wh.deleteDir(p.path, true); + wh.deleteDir(p.path, true, mustPurge); try { - deleteParentRecursive(p.path.getParent(), p.partValSize - 1); + deleteParentRecursive(p.path.getParent(), p.partValSize - 1, mustPurge); } catch (IOException ex) { LOG.warn("Error from deleteParentRecursive", ex); throw new MetaException("Failed to delete parent: " + ex.getMessage()); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 4ebf987..436b2f4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -739,6 +739,19 @@ public boolean dropPartition(String dbName, String tableName, String partName, b return dropPartition(dbName, tableName, partName, deleteData, null); } + private static EnvironmentContext getEnvironmentContextWithIfPurgeSet() { + Map warehouseOptions = new HashMap(); + warehouseOptions.put("ifPurge", "TRUE"); + return new EnvironmentContext(warehouseOptions); + } + + public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException { + + return dropPartition(dbName, tableName, partName, deleteData, + ifPurge? getEnvironmentContextWithIfPurgeSet() : null); + } + public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData, EnvironmentContext envContext) throws NoSuchObjectException, MetaException, TException { return client.drop_partition_by_name_with_environment_context(dbName, tableName, partName, @@ -765,6 +778,15 @@ public boolean dropPartition(String db_name, String tbl_name, return dropPartition(db_name, tbl_name, part_vals, deleteData, null); } + @Override + public boolean dropPartition(String db_name, String tbl_name, + List part_vals, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException { + + return dropPartition(db_name, tbl_name, part_vals, deleteData, + ifPurge? getEnvironmentContextWithIfPurgeSet() : null); + } + public boolean dropPartition(String db_name, String tbl_name, List part_vals, boolean deleteData, EnvironmentContext envContext) throws NoSuchObjectException, MetaException, TException { @@ -772,10 +794,16 @@ public boolean dropPartition(String db_name, String tbl_name, List part_ envContext); } - @Override public List dropPartitions(String dbName, String tblName, List> partExprs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws NoSuchObjectException, MetaException, TException { + return dropPartitions(dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists, false); + } + + @Override + public List dropPartitions(String dbName, String tblName, + List> partExprs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws NoSuchObjectException, MetaException, TException { RequestPartsSpec rps = new RequestPartsSpec(); List exprs = new ArrayList(partExprs.size()); for (ObjectPair partExpr : partExprs) { @@ -790,6 +818,10 @@ public boolean dropPartition(String db_name, String tbl_name, List part_ req.setIgnoreProtection(ignoreProtection); req.setNeedResult(true); req.setIfExists(ifExists); + if (ifPurge) { + LOG.info("Dropped partitions will be purged!"); + req.setEnvironmentContext(getEnvironmentContextWithIfPurgeSet()); + } return client.drop_partitions_req(req).getPartitions(); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 11b0336..5267f99 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -656,9 +656,30 @@ boolean dropPartition(String db_name, String tbl_name, List> partExprs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws NoSuchObjectException, MetaException, TException; + List dropPartitions(String dbName, String tblName, + List> partExprs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws NoSuchObjectException, MetaException, TException; + boolean dropPartition(String db_name, String tbl_name, String name, boolean deleteData) throws NoSuchObjectException, MetaException, TException; + + /** + * Method to dropPartitions() with the option to purge the partition data directly, + * rather than to move data to trash. + * @param db_name Name of the database. + * @param tbl_name Name of the table. + * @param part_vals Specification of the partitions being dropped. + * @param deleteData Whether or not to delete data. + * @param ifPurge Whether or not to purge data directly. + * @return True (if partitions are dropped), else false. + * @throws NoSuchObjectException + * @throws MetaException + * @throws TException + */ + public boolean dropPartition(String db_name, String tbl_name, + List part_vals, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException; /** * updates a partition to new partition * diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 1d0ed51..2ec5a9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -3640,7 +3640,7 @@ private void dropTableOrPartitions(Hive db, DropTableDesc dropTbl) throws HiveEx private void dropPartitions(Hive db, Table tbl, DropTableDesc dropTbl) throws HiveException { // ifExists is currently verified in DDLSemanticAnalyzer List droppedParts = db.dropPartitions(dropTbl.getTableName(), - dropTbl.getPartSpecs(), true, dropTbl.getIgnoreProtection(), true); + dropTbl.getPartSpecs(), true, dropTbl.getIgnoreProtection(), true, dropTbl.getIfPurge()); for (Partition partition : droppedParts) { console.printInfo("Dropped the partition " + partition.getName()); // We have already locked the table, don't lock the partitions. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index cd3d349..f5f73be 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1785,8 +1785,13 @@ public boolean dropPartition(String tblName, List part_vals, boolean del public boolean dropPartition(String db_name, String tbl_name, List part_vals, boolean deleteData) throws HiveException { + return dropPartition(db_name, tbl_name, part_vals, deleteData, false); + } + + public boolean dropPartition(String dbName, String tableName, List partVals, + boolean deleteData, boolean ifPurge) throws HiveException { try { - return getMSC().dropPartition(db_name, tbl_name, part_vals, deleteData); + return getMSC().dropPartition(dbName, tableName, partVals, deleteData, ifPurge); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e); } catch (Exception e) { @@ -1796,15 +1801,25 @@ public boolean dropPartition(String db_name, String tbl_name, public List dropPartitions(String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws HiveException { + return dropPartitions(tblName, partSpecs, deleteData, ignoreProtection, ifExists, false); + } + + public List dropPartitions(String tblName, List partSpecs, + boolean deleteData, boolean ignoreProtection, boolean ifExists, boolean ifPurge) throws HiveException { String[] names = Utilities.getDbTableName(tblName); return dropPartitions( - names[0], names[1], partSpecs, deleteData, ignoreProtection, ifExists); + names[0], names[1], partSpecs, deleteData, ignoreProtection, ifExists, ifPurge); } public List dropPartitions(String dbName, String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws HiveException { - //TODO: add support for ifPurge + return dropPartitions(dbName, tblName, partSpecs, deleteData, ignoreProtection, ifExists, false); + } + + public List dropPartitions(String dbName, String tblName, + List partSpecs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws HiveException { try { Table tbl = getTable(dbName, tblName); List> partExprs = @@ -1814,7 +1829,7 @@ public boolean dropPartition(String db_name, String tbl_name, Utilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } List tParts = getMSC().dropPartitions( - dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists); + dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists, ifPurge); return convertFromMetastore(tbl, tParts, null); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index da7d082..181cb2a 100755 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.regex.Pattern; +import com.google.common.collect.ImmutableMap; import junit.framework.TestCase; import org.apache.hadoop.fs.FileStatus; @@ -426,6 +427,100 @@ public void testDropTableTrash() throws Throwable { } } + private FileStatus[] getTrashContents() throws Exception { + FileSystem fs = FileSystem.get(hiveConf); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(hiveConf, fs); + return fs.globStatus(trashDir.suffix("/*")); + } + + private Table createPartitionedTable(String dbName, String tableName) throws Exception { + try { + + hm.dropTable(dbName, tableName); + hm.createTable(tableName, + Arrays.asList("key", "value"), // Data columns. + Arrays.asList("ds", "hr"), // Partition columns. + TextInputFormat.class, + HiveIgnoreKeyTextOutputFormat.class); + return hm.getTable(dbName, tableName); + } + catch (Exception exception) { + fail("Unable to drop and create table " + dbName + "." + tableName + + " because " + StringUtils.stringifyException(exception)); + throw exception; + } + } + + private void cleanUpTableQuietly(String dbName, String tableName) { + try { + hm.dropTable(dbName, tableName, true, true, true); + } + catch(Exception exception) { + fail("Unexpected exception: " + StringUtils.stringifyException(exception)); + } + } + + /** + * Test for PURGE support for dropping partitions. + * 1. Drop partitions without PURGE, and check that the data isn't moved to Trash. + * 2. Drop partitions with PURGE, and check that the data is moved to Trash. + * @throws Exception on failure. + */ + public void testDropPartitionsWithPurge() throws Exception { + String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME; + String tableName = "table_for_testDropPartitionsWithPurge"; + + try { + + Map partitionSpec = new ImmutableMap.Builder() + .put("ds", "20141216") + .put("hr", "12") + .build(); + + int trashSizeBeforeDrop = getTrashContents().length; + + Table table = createPartitionedTable(dbName, tableName); + hm.createPartition(table, partitionSpec); + + Partition partition = hm.getPartition(table, partitionSpec, false); + assertNotNull("Newly created partition shouldn't be null!", partition); + + hm.dropPartition(dbName, tableName, + partition.getValues(), + true,// Delete data. + true // Purge. + ); + + int trashSizeAfterDropPurge = getTrashContents().length; + + assertEquals("After dropPartitions(purge), trash should've remained unchanged!", + trashSizeBeforeDrop, trashSizeAfterDropPurge); + + // Repeat, and drop partition without purge. + hm.createPartition(table, partitionSpec); + + partition = hm.getPartition(table, partitionSpec, false); + assertNotNull("Newly created partition shouldn't be null!", partition); + + hm.dropPartition(dbName, tableName, + partition.getValues(), + true,// Delete data. + false // Purge. + ); + + int trashSizeWithoutPurge = getTrashContents().length; + + assertEquals("After dropPartitions(noPurge), data should've gone to trash!", + trashSizeBeforeDrop, trashSizeWithoutPurge); + + } + catch (Exception e) { + fail("Unexpected exception: " + StringUtils.stringifyException(e)); + } + finally { + cleanUpTableQuietly(dbName, tableName); + } + } public void testPartition() throws Throwable { try {