Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision ) @@ -824,6 +824,23 @@ /** * Drops table along with the data in it. If the table doesn't exist then it + * is a no-op, if skipTrash option is specified it is passed to + * hdfs command that removes table data from warehouse to skip trash. + * + * @param tableName + * table to drop + * @param skipTrash + * skips Trash while removing data from warehouse + * @throws HiveException + * thrown if the drop fails + */ + public void dropTable(String tableName, boolean skipTrash) throws HiveException { + Table t = newTable(tableName); + dropTable(t.getDbName(), t.getTableName(), true, true, skipTrash); + } + + /** + * Drops table along with the data in it. If the table doesn't exist then it * is a no-op * * @param tableName @@ -832,8 +849,7 @@ * thrown if the drop fails */ public void dropTable(String tableName) throws HiveException { - Table t = newTable(tableName); - dropTable(t.getDbName(), t.getTableName(), true, true); + dropTable(tableName, false); } /** @@ -860,13 +876,15 @@ * deletes the underlying data along with metadata * @param ignoreUnknownTab * an exception if thrown if this is falser and table doesn't exist + * @param skipTrash + * skips Trash while removing data from warehouse * @throws HiveException */ public void dropTable(String dbName, String tableName, boolean deleteData, - boolean ignoreUnknownTab) throws HiveException { + boolean ignoreUnknownTab, boolean skipTrash) throws HiveException { try { - getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab); + getMSC().dropTable(dbName, tableName, deleteData, ignoreUnknownTab, skipTrash); } catch (NoSuchObjectException e) { if (!ignoreUnknownTab) { throw new HiveException(e); @@ -874,6 +892,22 @@ } catch (Exception e) { throw new HiveException(e); } + } + + /** + * Drops the table. + * + * @param dbName + * @param tableName + * @param deleteData + * deletes the underlying data along with metadata + * @param ignoreUnknownTab + * an exception if thrown if this is falser and table doesn't exist + * @throws HiveException + */ + public void dropTable(String dbName, String tableName, boolean deleteData, + boolean ignoreUnknownTab) throws HiveException { + dropTable(dbName, tableName, deleteData, ignoreUnknownTab, false); } public HiveConf getConf() { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision ) @@ -827,8 +827,10 @@ outputs.add(new WriteEntity(tab)); } + boolean skipTrash = (ast.getFirstChildWithType(HiveParser.TOK_SKIPTRASH) != null); DropTableDesc dropTblDesc = new DropTableDesc( - tableName, expectView, ifExists, true); + tableName, expectView, ifExists, skipTrash, true); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf)); } Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision ) @@ -3326,7 +3326,7 @@ } // drop the table - db.dropTable(dropTbl.getTableName()); + db.dropTable(dropTbl.getTableName(), dropTbl.getSkipTrash()); if (tbl != null) { work.getOutputs().add(new WriteEntity(tbl)); } Index: metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (revision ) @@ -163,6 +163,26 @@ NoSuchObjectException; /** + * Drop the table. + * + * @param dbname + * The database for this table + * @param tableName + * The table to drop + * @param skipTrash + * skip trash while removing table data from warehouse + * @throws MetaException + * Could not drop table properly. + * @throws NoSuchObjectException + * The table wasn't found. + * @throws TException + * A thrift communication error occurred + * @throws ExistingDependentsException + */ + public void dropTable(String dbname, String tableName, boolean deleteData, + boolean ignoreUknownTab, boolean skipTrash) throws MetaException, TException, + NoSuchObjectException; + /** * Drop the table in the DEFAULT database. * * @param tableName Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (revision ) @@ -1,3 +1,5 @@ + KW_LIMIT num=Number -> ^(TOK_LIMIT $num) + ; /** Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with @@ -301,6 +303,7 @@ TOK_WINDOWRANGE; TOK_IGNOREPROTECTION; TOK_EXCHANGEPARTITION; +TOK_SKIPTRASH; } @@ -636,6 +639,13 @@ -> ^(TOK_IFEXISTS) ; +skipTrash +@init { msgs.push("skipTrash clause"); } +@after { msgs.pop(); } + : KW_SKIPTRASH + -> ^(TOK_SKIPTRASH) + ; + restrictOrCascade @init { msgs.push("restrict or cascade clause"); } @after { msgs.pop(); } @@ -843,7 +853,7 @@ dropTableStatement @init { msgs.push("drop statement"); } @after { msgs.pop(); } - : KW_DROP KW_TABLE ifExists? tableName -> ^(TOK_DROPTABLE tableName ifExists?) + : KW_DROP KW_TABLE ifExists? skipTrash? tableName -> ^(TOK_DROPTABLE tableName ifExists? skipTrash?) ; alterStatement Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision ) @@ -1200,7 +1200,7 @@ // Delete the data in the partitions which have other locations deletePartitionData(partPaths); // Delete the data in the table - deleteTableData(tblPath); + deleteTableData(tblPath, envContext); // ok even if the data is not deleted } for (MetaStoreEventListener listener : listeners) { @@ -1217,14 +1217,28 @@ * @param tablePath */ private void deleteTableData(Path tablePath) { + deleteTableData(tablePath, null); + } + + /** + * Deletes the data in a table's location, if it fails logs an error + * + * @param tablePath + * @param envContext + */ + private void deleteTableData(Path tablePath, EnvironmentContext envContext) { - if (tablePath != null) { - try { + if (tablePath != null) { + try { - wh.deleteDir(tablePath, true); + boolean skipTrash = false; + if (envContext != null){ + skipTrash = Boolean.parseBoolean(envContext.getProperties().get("skipTrash")); + } + wh.deleteDir(tablePath, true, skipTrash); - } catch (Exception e) { - LOG.error("Failed to delete table directory: " + tablePath + - " " + e.getMessage()); - } - } + } catch (Exception e) { + LOG.error("Failed to delete table directory: " + tablePath + + " " + e.getMessage()); + } + } } /** @@ -1316,7 +1330,6 @@ drop_table_with_environment_context(dbname, name, deleteData, null); } - @Override public void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData, final EnvironmentContext envContext) throws NoSuchObjectException, MetaException { Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (revision ) @@ -28,12 +28,7 @@ import java.lang.reflect.Proxy; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.HashMap; import javax.security.auth.login.LoginException; @@ -645,6 +640,34 @@ boolean ignoreUnknownTab) throws MetaException, TException, NoSuchObjectException, UnsupportedOperationException { dropTable(dbname, name, deleteData, ignoreUnknownTab, null); + } + + /** + * @param dbname + * @param name + * @param deleteData + * delete the underlying data or just delete the table in metadata + * @param skipTrash + * skip trash while removing table data from warehouse + * @throws NoSuchObjectException + * @throws ExistingDependentsException + * @throws MetaException + * @throws TException + * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#drop_table(java.lang.String, + * java.lang.String, boolean) + */ + public void dropTable(String dbname, String name, boolean deleteData, + boolean ignoreUnknownTab, boolean skipTrash) throws MetaException, TException, + NoSuchObjectException, UnsupportedOperationException { + //build new environmentContext with skipTrash; + EnvironmentContext mapRedOptionEnvContext = null; + if(skipTrash){ + Map mapRedOption = null; + mapRedOption = new HashMap(); + mapRedOption.put("skipTrash", "TRUE"); + mapRedOptionEnvContext = new EnvironmentContext(mapRedOption); + } + dropTable(dbname, name, deleteData, ignoreUnknownTab, mapRedOptionEnvContext); } public void dropTable(String dbname, String name, boolean deleteData, Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision ) @@ -217,8 +217,12 @@ } public boolean deleteDir(Path f, boolean recursive) throws MetaException { + return deleteDir(f, recursive, false); + } + + public boolean deleteDir(Path f, boolean recursive, boolean skipTrash) throws MetaException { FileSystem fs = getFs(f); - return fsHandler.deleteDir(fs, f, recursive, conf); + return fsHandler.deleteDir(fs, f, recursive, skipTrash, conf); } public boolean isWritable(Path path) throws IOException { Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreFsImpl.java (revision ) @@ -36,12 +36,12 @@ @Override public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException { + boolean skipTrash, Configuration conf) throws MetaException { LOG.info("deleting " + f); HadoopShims hadoopShim = ShimLoader.getHadoopShims(); try { - if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) { + if (!skipTrash && hadoopShim.moveToAppropriateTrash(fs, f, conf)) { LOG.info("Moved to trash: " + f); return true; } Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java (date 1391455406000) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreFS.java (revision ) @@ -33,10 +33,11 @@ * * @param f * @param recursive + * @param skipTrash * @return true on success * @throws MetaException */ public boolean deleteDir(FileSystem fs, Path f, boolean recursive, - Configuration conf) throws MetaException; + boolean skipTrash, Configuration conf) throws MetaException; } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (revision ) @@ -30,6 +30,7 @@ KW_IF : 'IF'; KW_EXISTS : 'EXISTS'; +KW_SKIPTRASH : 'SKIPTRASH'; KW_ASC : 'ASC'; KW_DESC : 'DESC'; Index: ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (date 1391455406000) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (revision ) @@ -34,6 +34,7 @@ ArrayList partSpecs; boolean expectView; boolean ifExists; + boolean skipTrash; boolean ignoreProtection; boolean stringPartitionColumns; // This is due to JDO not working very well with // non-string partition columns. @@ -44,13 +45,15 @@ /** * @param tableName + * @param skipTrash */ public DropTableDesc(String tableName, boolean expectView, - boolean ifExists, boolean stringPartitionColumns) { + boolean ifExists, boolean skipTrash, boolean stringPartitionColumns) { this.tableName = tableName; partSpecs = null; this.expectView = expectView; this.ifExists = ifExists; + this.skipTrash = skipTrash; this.ignoreProtection = false; this.stringPartitionColumns = stringPartitionColumns; } @@ -144,6 +147,17 @@ public void setIfExists(boolean ifExists) { this.ifExists = ifExists; } + + /** + * @return whether SKIPTRASH was specified + */ + public boolean getSkipTrash() { return skipTrash; } + + /** + * @param skipTrash + * set whether SKIPTRASH was specified + */ + public void setSkipTrash(boolean skipTrash) { this.skipTrash = skipTrash; } public boolean isStringPartitionColumns() { return stringPartitionColumns;