diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index e89eded..8dcd4cf 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -559,13 +559,7 @@ public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) thr LOG.info("deleting " + f); HadoopShims hadoopShim = ShimLoader.getHadoopShims(); - boolean skipTrash = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH); - - if (skipTrash) { - LOG.info("Not moving "+ f +" to trash due to configuration " + - HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH + " is set to true."); - } else if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) { + if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) { LOG.info("Moved to trash: " + f); return true; } diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7932a3d..9244c3a 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -863,7 +863,6 @@ HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false), HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false), HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", false), - HIVE_WAREHOUSE_DATA_SKIPTRASH("hive.warehouse.data.skipTrash", false), // whether insert into external tables is allowed HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true), diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 487d292..345163e 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1380,9 +1380,9 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin ms.rollbackTransaction(); } else if (deleteData && !isExternal) { // Delete the data in the partitions which have other locations - deletePartitionData(partPaths); + deletePartitionData(partPaths, envContext); // Delete the data in the table - deleteTableData(tblPath); + deleteTableData(tblPath, envContext); // ok even if the data is not deleted } for (MetaStoreEventListener listener : listeners) { @@ -1399,9 +1399,23 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin * @param tablePath */ private void deleteTableData(Path tablePath) { + deleteTableData(tablePath, null); + } + + /** + * Deletes the data in a table's location, if it fails logs an error + * + * @param tablePath + * @param envContext + */ + private void deleteTableData(Path tablePath, EnvironmentContext envContext) { if (tablePath != null) { try { - wh.deleteDir(tablePath, true); + boolean ifPurge = false; + if (envContext != null){ + ifPurge = Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + } + wh.deleteDir(tablePath, true, ifPurge); } catch (Exception e) { LOG.error("Failed to delete table directory: " + tablePath + " " + e.getMessage()); @@ -1416,10 +1430,25 @@ private void deleteTableData(Path tablePath) { * @param partPaths */ private void deletePartitionData(List partPaths) { + deletePartitionData(partPaths, null); + } + + /** + * Give a list of partitions' locations, tries to delete each one + * and for each that fails logs an error. + * + * @param partPaths + * @param envContext + */ + private void deletePartitionData(List partPaths, EnvironmentContext envContext) { if (partPaths != null && !partPaths.isEmpty()) { for (Path partPath : partPaths) { try { - wh.deleteDir(partPath, true); + boolean ifPurge = false; + if (envContext != null){ + ifPurge = Boolean.parseBoolean(envContext.getProperties().get("ifPurge")); + } + wh.deleteDir(partPath, true, ifPurge); } catch (Exception e) { LOG.error("Failed to delete partition directory: " + partPath + " " + e.getMessage()); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 664dccd..2dadaa5 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -38,6 +38,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.HashMap; import java.util.Random; import java.util.Set; @@ -750,6 +751,34 @@ public void dropTable(String dbname, String name, boolean deleteData, dropTable(dbname, name, deleteData, ignoreUnknownTab, null); } + /** + * @param dbname + * @param name + * @param deleteData + * delete the underlying data or just delete the table in metadata + * @param ifPurge + * completely purge the table skipping trash while removing data from warehouse + * @throws NoSuchObjectException + * @throws ExistingDependentsException + * @throws MetaException + * @throws TException + * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, + * java.lang.String, boolean) + */ + public void dropTable(String dbname, String name, boolean deleteData, + boolean ignoreUnknownTab, boolean ifPurge) throws MetaException, TException, + NoSuchObjectException, UnsupportedOperationException { + //build new environmentContext with ifPurge; + EnvironmentContext envContext = null; + if(ifPurge){ + Map warehouseOptions = null; + warehouseOptions = new HashMap(); + warehouseOptions.put("ifPurge", "TRUE"); + envContext = new EnvironmentContext(warehouseOptions); + } + dropTable(dbname, name, deleteData, ignoreUnknownTab, envContext); + } + public void dropTable(String dbname, String name, boolean deleteData, boolean ignoreUnknownTab, EnvironmentContext envContext) throws MetaException, TException, NoSuchObjectException, UnsupportedOperationException { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java index e32b096..c3598db 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java @@ -37,17 +37,13 @@ @Override public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException { + boolean ifPurge, Configuration conf) throws MetaException { LOG.info("deleting " + f); HadoopShims hadoopShim = ShimLoader.getHadoopShims(); - boolean skipTrash = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH); - try { - if (skipTrash) { - LOG.info("Not moving"+ f +" to trash due to configuration " + - HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH + " is set to true."); + if (ifPurge) { + LOG.info("Not moving "+ f +" to trash"); } else if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) { LOG.info("Moved to trash: " + f); return true; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 0c2209b..39be5be 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -197,6 +197,26 @@ public void dropTable(String dbname, String tableName, boolean deleteData, NoSuchObjectException; /** + * Drop the table. + * + * @param dbname + * The database for this table + * @param tableName + * The table to drop + * @param ifPurge + * completely purge the table skipping trash while removing data from warehouse + * @throws MetaException + * Could not drop table properly. + * @throws NoSuchObjectException + * The table wasn't found. + * @throws TException + * A thrift communication error occurred + * @throws ExistingDependentsException + */ + public void dropTable(String dbname, String tableName, boolean deleteData, + boolean ignoreUknownTab, boolean ifPurge) throws MetaException, TException, + NoSuchObjectException; + /** * Drop the table in the DEFAULT database. * * @param tableName diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java index a141793..c248409 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java @@ -33,10 +33,11 @@ * * @param f * @param recursive + * @param ifPurge * @return true on success * @throws MetaException */ public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException; + boolean ifPurge, Configuration conf) throws MetaException; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index 9e8d912..3ac7fd7 100755 --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -209,8 +209,12 @@ public boolean renameDir(Path sourcePath, Path destPath, boolean inheritPerms) t } public boolean deleteDir(Path f, boolean recursive) throws MetaException { + return deleteDir(f, recursive, false); + } + + public boolean deleteDir(Path f, boolean recursive, boolean ifPurge) throws MetaException { FileSystem fs = getFs(f); - return fsHandler.deleteDir(fs, f, recursive, conf); + return fsHandler.deleteDir(fs, f, recursive, ifPurge, conf); } public boolean isEmpty(Path path) throws IOException, MetaException { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index fad5ed3..1e93de3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -4011,7 +4011,7 @@ private void dropTable(Hive db, Table tbl, DropTableDesc dropTbl) throws HiveExc } // drop the table - db.dropTable(dropTbl.getTableName()); + db.dropTable(dropTbl.getTableName(), dropTbl.getIfPurge()); if (tbl != null) { // We have already locked the table in DDLSemenaticAnalyzer, don't do it again here work.getOutputs().add(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index a988b44..b417f90 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -854,6 +854,23 @@ public boolean dropIndex(String db_name, String tbl_name, String index_name, boo /** * Drops table along with the data in it. If the table doesn't exist then it + * is a no-op, if ifPurge option is specified it is passed to + * hdfs command that removes table data from warehouse to skip trash. + * + * @param tableName + * table to drop + * @param ifPurge + * completely purge the table skipping trash while removing data from warehouse + * @throws HiveException + * thrown if the drop fails + */ + public void dropTable(String tableName, boolean ifPurge) throws HiveException { + Table t = newTable(tableName); + dropTable(t.getDbName(), t.getTableName(), true, true, ifPurge); + } + + /** + * Drops table along with the data in it. If the table doesn't exist then it * is a no-op * * @param tableName @@ -862,8 +879,7 @@ public boolean dropIndex(String db_name, String tbl_name, String index_name, boo * thrown if the drop fails */ public void dropTable(String tableName) throws HiveException { - Table t = newTable(tableName); - dropTable(t.getDbName(), t.getTableName(), true, true); + dropTable(tableName, false); } /** @@ -890,13 +906,15 @@ public void dropTable(String dbName, String tableName) throws HiveException { * deletes the underlying data along with metadata * @param ignoreUnknownTab * an exception if thrown if this is falser and table doesn't exist + * @param ifPurge + * completely purge the table skipping trash while removing data from warehouse * @throws HiveException */ public void dropTable(String dbName, String tableName, boolean deleteData, - boolean ignoreUnknownTab) throws HiveException { + boolean ignoreUnknownTab, boolean ifPurge) throws HiveException { try { - getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab); + getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab, ifPurge); } catch (NoSuchObjectException e) { if (!ignoreUnknownTab) { throw new HiveException(e); @@ -906,6 +924,22 @@ public void dropTable(String dbName, String tableName, boolean deleteData, } } + /** + * Drops the table. + * + * @param dbName + * @param tableName + * @param deleteData + * deletes the underlying data along with metadata + * @param ignoreUnknownTab + * an exception if thrown if this is falser and table doesn't exist + * @throws HiveException + */ + public void dropTable(String dbName, String tableName, boolean deleteData, + boolean ignoreUnknownTab) throws HiveException { + dropTable(dbName, tableName, deleteData, ignoreUnknownTab, false); + } + public HiveConf getConf() { return (conf); } @@ -1706,14 +1740,20 @@ public boolean dropPartition(String db_name, String tbl_name, public List dropPartitions(String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, boolean ifExists) throws HiveException { + return dropPartitions( + tblName, partSpecs, deleteData, ignoreProtection, ifExists, false); + } + + public List dropPartitions(String tblName, List partSpecs, + boolean deleteData, boolean ignoreProtection, boolean ifExists, boolean ifPurge) throws HiveException { Table t = newTable(tblName); return dropPartitions( - t.getDbName(), t.getTableName(), partSpecs, deleteData, ignoreProtection, ifExists); + t.getDbName(), t.getTableName(), partSpecs, deleteData, ignoreProtection, ifExists, ifPurge); } public List dropPartitions(String dbName, String tblName, List partSpecs, boolean deleteData, boolean ignoreProtection, - boolean ifExists) throws HiveException { + boolean ifExists, boolean ifPurge) throws HiveException { try { Table tbl = getTable(dbName, tblName); List> partExprs = diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index 1208bce..81d54d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -813,7 +813,8 @@ private void analyzeDropTable(ASTNode ast, boolean expectView) outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_EXCLUSIVE)); } - DropTableDesc dropTblDesc = new DropTableDesc(tableName, expectView, ifExists); + boolean ifPurge = (ast.getFirstChildWithType(HiveParser.KW_PURGE) != null); + DropTableDesc dropTblDesc = new DropTableDesc(tableName, expectView, ifExists, ifPurge); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index f934ac4..bb324eb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -916,7 +916,7 @@ dropIndexStatement dropTableStatement @init { pushMsg("drop statement", state); } @after { popMsg(state); } - : KW_DROP KW_TABLE ifExists? tableName -> ^(TOK_DROPTABLE tableName ifExists?) + : KW_DROP KW_TABLE ifExists? tableName KW_PURGE?-> ^(TOK_DROPTABLE tableName ifExists? KW_PURGE?) ; alterStatement diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java index ba30e1f..f249a20 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java @@ -53,18 +53,21 @@ public int getPrefixLength() { boolean expectView; boolean ifExists; boolean ignoreProtection; + boolean ifPurge; public DropTableDesc() { } /** * @param tableName + * @param ifPurge */ - public DropTableDesc(String tableName, boolean expectView, boolean ifExists) { + public DropTableDesc(String tableName, boolean expectView, boolean ifExists, boolean ifPurge) { this.tableName = tableName; this.partSpecs = null; this.expectView = expectView; this.ifExists = ifExists; + this.ifPurge = ifPurge; this.ignoreProtection = false; } @@ -149,4 +152,15 @@ public boolean getIfExists() { public void setIfExists(boolean ifExists) { this.ifExists = ifExists; } + + /** + * @return whether Purge was specified + */ + public boolean getIfPurge() { return ifPurge; } + + /** + * @param ifPurge + * set whether Purge was specified + */ + public void setIfPurge(boolean ifPurge) { this.ifPurge = ifPurge; } }