diff --git .gitignore .gitignore index d0c97d1..05d1e03 100644 --- .gitignore +++ .gitignore @@ -25,3 +25,5 @@ hcatalog/core/target hcatalog/webhcat/java-client/target hcatalog/storage-handlers/hbase/target hcatalog/webhcat/svr/target +lib/ +testlibs/ diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 487d292..f3bcacc 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1380,9 +1380,9 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin ms.rollbackTransaction(); } else if (deleteData && !isExternal) { // Delete the data in the partitions which have other locations - deletePartitionData(partPaths); + deletePartitionData(partPaths, envContext); // Delete the data in the table - deleteTableData(tblPath); + deleteTableData(tblPath, envContext); // ok even if the data is not deleted } for (MetaStoreEventListener listener : listeners) { @@ -1399,9 +1399,24 @@ private void drop_table_core(final RawStore ms, final String dbname, final Strin * @param tablePath */ private void deleteTableData(Path tablePath) { + deleteTableData(tablePath, null); + } + + /** + * Deletes the data in a table's location, if it fails logs an error + * + * @param tablePath + * @param envContext + */ + private void deleteTableData(Path tablePath, EnvironmentContext envContext) { if (tablePath != null) { try { - wh.deleteDir(tablePath, true); + boolean skipTrash = false; + if (envContext != null) { + skipTrash = Boolean.parseBoolean( + envContext.getProperties().get(ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH.varname)); + } + wh.deleteDir(tablePath, true, skipTrash); } catch (Exception e) { LOG.error("Failed to delete table directory: " + tablePath + " " + e.getMessage()); @@ -1429,6 +1444,31 @@ private void deletePartitionData(List partPaths) { } /** + * Give a list of partitions' locations, tries to delete each one + * and for each that fails logs an error. + * + * @param partPaths + * @param envContext + */ + private void deletePartitionData(List partPaths, EnvironmentContext envContext) { + if (partPaths != null && !partPaths.isEmpty()) { + for (Path partPath : partPaths) { + try { + boolean skipTrash = false; + if (envContext != null) { + skipTrash = Boolean.parseBoolean( + envContext.getProperties().get(ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH.varname)); + } + wh.deleteDir(partPath, true, skipTrash); + } catch (Exception e) { + LOG.error("Failed to delete partition directory: " + partPath + + " " + e.getMessage()); + } + } + } + } + + /** * Retrieves the partitions specified by partitionKeys. If checkLocation, for locations of * partitions which may not be subdirectories of tablePath checks to make the locations are * writable. diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 664dccd..10829d6 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -33,6 +33,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; @@ -750,6 +751,40 @@ public void dropTable(String dbname, String name, boolean deleteData, dropTable(dbname, name, deleteData, ignoreUnknownTab, null); } + /** + * @param dbname + * @param name + * @param deleteData + * delete the underlying data or just delete the table in metadata + * @param skipTrash skips trash while dropping table data from warehouse + * @throws NoSuchObjectException + * @throws MetaException + * @throws TException + * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, + * java.lang.String, boolean) + */ + @Override + public void dropTable(String dbname, String name, boolean deleteData, + boolean ignoreUnknownTab, boolean skipTrash) throws MetaException, TException, + NoSuchObjectException, UnsupportedOperationException { + // setting skipTash in envronment context for drop table + Map dropTableOptions = new HashMap(); + dropTableOptions.put(ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH.varname, String.valueOf(skipTrash)); + EnvironmentContext dropTableEnvContext = new EnvironmentContext(dropTableOptions); + dropTable(dbname, name, deleteData, ignoreUnknownTab, dropTableEnvContext); + } + + /** + * @param dbname + * @param name + * @param deleteData + * delete the underlying data or just delete the table in metadata + * @throws NoSuchObjectException + * @throws MetaException + * @throws TException + * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, + * java.lang.String, boolean) + */ public void dropTable(String dbname, String name, boolean deleteData, boolean ignoreUnknownTab, EnvironmentContext envContext) throws MetaException, TException, NoSuchObjectException, UnsupportedOperationException { diff --git metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java index e32b096..cb08b00 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java @@ -37,16 +37,13 @@ @Override public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException { + boolean skipTrash, Configuration conf) throws MetaException { LOG.info("deleting " + f); HadoopShims hadoopShim = ShimLoader.getHadoopShims(); - boolean skipTrash = HiveConf.getBoolVar(conf, - HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH); - try { if (skipTrash) { - LOG.info("Not moving"+ f +" to trash due to configuration " + + LOG.info("Not moving "+ f +" to trash because configuration " + HiveConf.ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH + " is set to true."); } else if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) { LOG.info("Moved to trash: " + f); diff --git metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 0c2209b..2cd874e 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -197,6 +197,26 @@ public void dropTable(String dbname, String tableName, boolean deleteData, NoSuchObjectException; /** + * Drop the table. + * + * @param dbname + * The database for this table + * @param tableName + * The table to drop + * @param skipTrash + * skips trash while dropping table data from warehouse + * @throws MetaException + * Could not drop table properly. + * @throws NoSuchObjectException + * The table wasn't found. + * @throws TException + * A thrift communication error occurred + */ + + public void dropTable(String dbname, String tableName, boolean deleteData, + boolean ignoreUknownTab, boolean skipTrash) throws MetaException, TException, NoSuchObjectException; + + /** * Drop the table in the DEFAULT database. * * @param tableName diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java index a141793..bdec1d9 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java @@ -33,10 +33,11 @@ * * @param f * @param recursive + * @param skipTrash * @return true on success * @throws MetaException */ public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException; + boolean skipTrash, Configuration conf) throws MetaException; } diff --git metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java index c1790b4..8cdb5b0 100755 --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -210,8 +210,12 @@ public boolean renameDir(Path sourcePath, Path destPath) throws MetaException { } public boolean deleteDir(Path f, boolean recursive) throws MetaException { + return deleteDir(f, recursive, false); + } + + public boolean deleteDir(Path f, boolean recursive, boolean skipTrash) throws MetaException { FileSystem fs = getFs(f); - return fsHandler.deleteDir(fs, f, recursive, conf); + return fsHandler.deleteDir(fs, f, recursive, skipTrash, conf); } public boolean isEmpty(Path path) throws IOException, MetaException { diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 3417474..cceaa05 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -896,7 +896,8 @@ public void dropTable(String dbName, String tableName, boolean deleteData, boolean ignoreUnknownTab) throws HiveException { try { - getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab); + boolean skipTrash = conf.getBoolVar(ConfVars.HIVE_WAREHOUSE_DATA_SKIPTRASH); + getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab, skipTrash); } catch (NoSuchObjectException e) { if (!ignoreUnknownTab) { throw new HiveException(e);