diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 2547da7..b805294 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -2551,15 +2551,24 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na ms.rollbackTransaction(); } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { if (tbl != null && !isExternal(tbl)) { + // Data needs deletion. Check if trash may be skipped. + boolean mustPurge = (envContext != null) + && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + if (mustPurge) { + LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash."); + } + else { + LOG.info("dropPartition() will move " + partPath + " to trash-directory."); + } // Archived partitions have har:/to_har_file as their location. // The original directory was saved in params if (isArchived) { assert (archiveParentDir != null); - wh.deleteDir(archiveParentDir, true); + wh.deleteDir(archiveParentDir, true, mustPurge); } else { assert (partPath != null); - wh.deleteDir(partPath, true); - deleteParentRecursive(partPath.getParent(), part_vals.size() - 1); + wh.deleteDir(partPath, true, mustPurge); + deleteParentRecursive(partPath.getParent(), part_vals.size() - 1, mustPurge); } // ok even if the data is not deleted } @@ -2574,10 +2583,10 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na return true; } - private void deleteParentRecursive(Path parent, int depth) throws IOException, MetaException { + private void deleteParentRecursive(Path parent, int depth, boolean mustPurge) throws IOException, MetaException { if (depth > 0 && parent != null && wh.isWritable(parent) && wh.isEmpty(parent)) { - wh.deleteDir(parent, true); - deleteParentRecursive(parent.getParent(), depth - 1); + wh.deleteDir(parent, true, mustPurge); + deleteParentRecursive(parent.getParent(), depth - 1, mustPurge); } } @@ -2704,15 +2713,21 @@ public DropPartitionsResult drop_partitions_req( if (!success) { ms.rollbackTransaction(); } else if (deleteData && !isExternal(tbl)) { + // Data needs deletion. Check if trash may be skipped. + boolean mustPurge = (envContext != null) + && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + LOG.info( mustPurge? + "dropPartition() will purge partition-directories directly, skipping trash." + : "dropPartition() will move partition-directories to trash-directory."); // Archived partitions have har:/to_har_file as their location. // The original directory was saved in params for (Path path : archToDelete) { - wh.deleteDir(path, true); + wh.deleteDir(path, true, mustPurge); } for (PathAndPartValSize p : dirsToDelete) { - wh.deleteDir(p.path, true); + wh.deleteDir(p.path, true, mustPurge); try { - deleteParentRecursive(p.path.getParent(), p.partValSize - 1); + deleteParentRecursive(p.path.getParent(), p.partValSize - 1, mustPurge); } catch (IOException ex) { LOG.warn("Error from deleteParentRecursive", ex); throw new MetaException("Failed to delete parent: " + ex.getMessage()); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 4ebf987..436b2f4 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -739,6 +739,19 @@ public boolean dropPartition(String dbName, String tableName, String partName, b return dropPartition(dbName, tableName, partName, deleteData, null); } + private static EnvironmentContext getEnvironmentContextWithIfPurgeSet() { + Map warehouseOptions = new HashMap(); + warehouseOptions.put("ifPurge", "TRUE"); + return new EnvironmentContext(warehouseOptions); + } + + public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException { + + return dropPartition(dbName, tableName, partName, deleteData, + ifPurge? getEnvironmentContextWithIfPurgeSet() : null); + } + public boolean dropPartition(String dbName, String tableName, String partName, boolean deleteData, EnvironmentContext envContext) throws NoSuchObjectException, MetaException, TException { return client.drop_partition_by_name_with_environment_context(dbName, tableName, partName, @@ -765,6 +778,15 @@ public boolean dropPartition(String db_name, String tbl_name, return dropPartition(db_name, tbl_name, part_vals, deleteData, null); } + @Override + public boolean dropPartition(String db_name, String tbl_name, + List part_vals, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException { + + return dropPartition(db_name, tbl_name, part_vals, deleteData, + ifPurge? getEnvironmentContextWithIfPurgeSet() : null); + } + public boolean dropPartition(String db_name, String tbl_name, List part_vals, boolean deleteData, EnvironmentContext envContext) throws NoSuchObjectException, MetaException, TException { @@ -772,10 +794,16 @@ public boolean dropPartition(String db_name, String tbl_name, List part_ envContext); } - @Override public List dropPartitions(String dbName, String tblName, List> partExprs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws NoSuchObjectException, MetaException, TException { + return dropPartitions(dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists, false); + } + + @Override + public List dropPartitions(String dbName, String tblName, + List> partExprs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws NoSuchObjectException, MetaException, TException { RequestPartsSpec rps = new RequestPartsSpec(); List exprs = new ArrayList(partExprs.size()); for (ObjectPair partExpr : partExprs) { @@ -790,6 +818,10 @@ public boolean dropPartition(String db_name, String tbl_name, List part_ req.setIgnoreProtection(ignoreProtection); req.setNeedResult(true); req.setIfExists(ifExists); + if (ifPurge) { + LOG.info("Dropped partitions will be purged!"); + req.setEnvironmentContext(getEnvironmentContextWithIfPurgeSet()); + } return client.drop_partitions_req(req).getPartitions(); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 11b0336..5267f99 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -656,9 +656,30 @@ boolean dropPartition(String db_name, String tbl_name, List> partExprs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws NoSuchObjectException, MetaException, TException; + List dropPartitions(String dbName, String tblName, + List> partExprs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws NoSuchObjectException, MetaException, TException; + boolean dropPartition(String db_name, String tbl_name, String name, boolean deleteData) throws NoSuchObjectException, MetaException, TException; + + /** + * Method to dropPartitions() with the option to purge the partition data directly, + * rather than to move data to trash. + * @param db_name Name of the database. + * @param tbl_name Name of the table. + * @param part_vals Specification of the partitions being dropped. + * @param deleteData Whether or not to delete data. + * @param ifPurge Whether or not to purge data directly. + * @return True (if partitions are dropped), else false. + * @throws NoSuchObjectException + * @throws MetaException + * @throws TException + */ + public boolean dropPartition(String db_name, String tbl_name, + List part_vals, boolean deleteData, boolean ifPurge) + throws NoSuchObjectException, MetaException, TException; /** * updates a partition to new partition * diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 1d0ed51..2ec5a9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -3640,7 +3640,7 @@ private void dropTableOrPartitions(Hive db, DropTableDesc dropTbl) throws HiveEx private void dropPartitions(Hive db, Table tbl, DropTableDesc dropTbl) throws HiveException { // ifExists is currently verified in DDLSemanticAnalyzer List droppedParts = db.dropPartitions(dropTbl.getTableName(), - dropTbl.getPartSpecs(), true, dropTbl.getIgnoreProtection(), true); + dropTbl.getPartSpecs(), true, dropTbl.getIgnoreProtection(), true, dropTbl.getIfPurge()); for (Partition partition : droppedParts) { console.printInfo("Dropped the partition " + partition.getName()); // We have already locked the table, don't lock the partitions. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index cd3d349..22ec48b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1796,15 +1796,25 @@ public boolean dropPartition(String db_name, String tbl_name, public List dropPartitions(String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws HiveException { + return dropPartitions(tblName, partSpecs, deleteData, ignoreProtection, ifExists, false); + } + + public List dropPartitions(String tblName, List partSpecs, + boolean deleteData, boolean ignoreProtection, boolean ifExists, boolean ifPurge) throws HiveException { String[] names = Utilities.getDbTableName(tblName); return dropPartitions( - names[0], names[1], partSpecs, deleteData, ignoreProtection, ifExists); + names[0], names[1], partSpecs, deleteData, ignoreProtection, ifExists, ifPurge); } public List dropPartitions(String dbName, String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws HiveException { - //TODO: add support for ifPurge + return dropPartitions(dbName, tblName, partSpecs, deleteData, ignoreProtection, ifExists, false); + } + + public List dropPartitions(String dbName, String tblName, + List partSpecs, boolean deleteData, boolean ignoreProtection, + boolean ifExists, boolean ifPurge) throws HiveException { try { Table tbl = getTable(dbName, tblName); List> partExprs = @@ -1814,7 +1824,7 @@ public boolean dropPartition(String db_name, String tbl_name, Utilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } List tParts = getMSC().dropPartitions( - dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists); + dbName, tblName, partExprs, deleteData, ignoreProtection, ifExists, ifPurge); return convertFromMetastore(tbl, tParts, null); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e);