Index: build-common.xml =================================================================== --- build-common.xml (revision 1001862) +++ build-common.xml (working copy) @@ -396,6 +396,7 @@ + @@ -425,6 +426,7 @@ + Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1001881) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -53,6 +53,7 @@ public static final HiveConf.ConfVars[] metaVars = { HiveConf.ConfVars.METASTOREDIRECTORY, HiveConf.ConfVars.METASTOREWAREHOUSE, + HiveConf.ConfVars.METASTORESCRATCHTRASH, HiveConf.ConfVars.METASTOREURIS, HiveConf.ConfVars.METATORETHRIFTRETRIES, HiveConf.ConfVars.METASTOREPWD, @@ -122,6 +123,7 @@ // something here! METASTOREDIRECTORY("hive.metastore.metadb.dir", ""), METASTOREWAREHOUSE("hive.metastore.warehouse.dir", ""), + METASTORESCRATCHTRASH("hive.metastore.warehouse.scratch.dir", ""), METASTOREURIS("hive.metastore.uris", ""), // Number of times to retry a connection to a Thrift metastore server METATORETHRIFTRETRIES("hive.metastore.connect.retries", 3), Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1001881) +++ conf/hive-default.xml (working copy) @@ -193,6 +193,12 @@ + hive.metastore.warehouse.scratch.dir + /user/hive/scratch_warehouse + location of scratch trash dir for the warehouse + + + hive.metastore.warehouse.dir /user/hive/warehouse location of default database for the warehouse Index: data/conf/hive-site.xml =================================================================== --- data/conf/hive-site.xml (revision 1001862) +++ data/conf/hive-site.xml (working copy) @@ -65,6 +65,12 @@ + hive.metastore.warehouse.scratch.dir + ${test.warehouse.scratch.dir} + + + + hive.metastore.metadb.dir file://${build.dir}/test/data/metadb/ Index: eclipse-templates/TestCliDriver.launchtemplate =================================================================== --- eclipse-templates/TestCliDriver.launchtemplate (revision 1001862) +++ eclipse-templates/TestCliDriver.launchtemplate (working copy) @@ -21,6 +21,6 @@ - + Index: eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate =================================================================== --- eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate (revision 1001862) +++ eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate (working copy) @@ -21,6 +21,6 @@ - + Index: eclipse-templates/TestHive.launchtemplate =================================================================== --- eclipse-templates/TestHive.launchtemplate (revision 1001862) +++ eclipse-templates/TestHive.launchtemplate (working copy) @@ -21,6 +21,6 @@ - + Index: eclipse-templates/TestHiveMetaStoreChecker.launchtemplate =================================================================== --- eclipse-templates/TestHiveMetaStoreChecker.launchtemplate (revision 1001862) +++ eclipse-templates/TestHiveMetaStoreChecker.launchtemplate (working copy) @@ -21,6 +21,6 @@ - + Index: eclipse-templates/TestJdbc.launchtemplate =================================================================== --- eclipse-templates/TestJdbc.launchtemplate (revision 1001862) +++ eclipse-templates/TestJdbc.launchtemplate (working copy) @@ -21,7 +21,7 @@ - + Index: eclipse-templates/TestRemoteHiveMetaStore.launchtemplate =================================================================== --- eclipse-templates/TestRemoteHiveMetaStore.launchtemplate (revision 1001862) +++ eclipse-templates/TestRemoteHiveMetaStore.launchtemplate (working copy) @@ -21,6 +21,6 @@ - + Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1001862) +++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy) @@ -850,6 +850,10 @@ Table tbl = null; isExternal = false; boolean isIndexTable = false; + + Path scratchTrash = null; + boolean movedToTrash = false; + try { ms.openTransaction(); // drop any partitions @@ -883,6 +887,10 @@ isExternal = isExternal(tbl); if (tbl.getSd().getLocation() != null) { tblPath = new Path(tbl.getSd().getLocation()); + if(deleteData && !isExternal) { + scratchTrash = wh.getScratchTrashPath(tblPath); + movedToTrash = wh.mvDirIfSrcExists(tblPath, scratchTrash); + } } if (!ms.dropTable(dbname, name)) { @@ -893,9 +901,17 @@ } finally { if (!success) { ms.rollbackTransaction(); - } else if (deleteData && (tblPath != null) && !isExternal) { - wh.deleteDir(tblPath, true); + if (movedToTrash) { + //recover data from hdfs + if (!wh.mvDirIfSrcExists(scratchTrash, tblPath)) { + throw new MetaException( + "Drop table failed, and data in Scratch Trash (" + + scratchTrash.toString() + ") can not be recovered!"); + } + } + } else if (movedToTrash) { // ok even if the data is not deleted + wh.deleteDir(scratchTrash, true); } } } @@ -1222,7 +1238,9 @@ Table tbl = null; Partition part = null; boolean isArchived = false; - Path archiveParentDir = null; + + Path scratchTrash = null; + boolean movedToTrash = false; try { ms.openTransaction(); @@ -1234,9 +1252,6 @@ } isArchived = MetaStoreUtils.isArchived(part); - if (isArchived) { - archiveParentDir = MetaStoreUtils.getOriginalLocation(part); - } if (part.getSd() == null || part.getSd().getLocation() == null) { throw new MetaException("Partition metadata is corrupted"); } @@ -1245,24 +1260,42 @@ } success = ms.commitTransaction(); partPath = new Path(part.getSd().getLocation()); + if (isArchived) { + // Archived partitions have har:/to_har_file as their location. + // The original directory was saved in params + partPath = MetaStoreUtils.getOriginalLocation(part); + } + tbl = get_table(db_name, tbl_name); + + if (tbl == null) { + throw new MetaException("Unable to load table " + tbl_name + + " from metastore."); + } + + if (partPath != null) { + if(deleteData && !isExternal(tbl)) { + scratchTrash = wh.getScratchTrashPath(partPath); + movedToTrash = wh.mvDirIfSrcExists(partPath, scratchTrash); + } + } + } finally { if (!success) { ms.rollbackTransaction(); - } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { - if (tbl != null && !isExternal(tbl)) { - // Archived partitions have har:/to_har_file as their location. - // The original directory was saved in params - if (isArchived) { - assert(archiveParentDir != null); - wh.deleteDir(archiveParentDir, true); - } else { - assert(partPath != null); - wh.deleteDir(partPath, true); + if (movedToTrash) { + //recover data from hdfs + if (!wh.mvDirIfSrcExists(scratchTrash, partPath)) { + throw new MetaException( + "Drop partition failed, and data in Scratch Trash (" + + scratchTrash.toString() + ") can not be recovered!"); } - // ok even if the data is not deleted } + } else if (movedToTrash) { + // ok even if the data is not deleted + wh.deleteDir(scratchTrash, true); } + } return true; } Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 1001862) +++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy) @@ -20,12 +20,16 @@ import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.GregorianCalendar; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,7 +55,15 @@ private final Configuration conf; private final String whRootString; + private final String whScratchString; + private Path whScratchTrashPath; + private static final String DATABASE_WAREHOUSE_SUFFIX = ".db"; + + private GregorianCalendar calendar = new GregorianCalendar(); + private SimpleDateFormat dateFormater = new SimpleDateFormat("yyyy-MM-dd"); + private Random rand = new Random(); + public static final Log LOG = LogFactory.getLog("hive.metastore.warehouse"); @@ -62,6 +74,11 @@ throw new MetaException(HiveConf.ConfVars.METASTOREWAREHOUSE.varname + " is not set in the config or blank"); } + whScratchString = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORESCRATCHTRASH); + if (StringUtils.isBlank(whScratchString)) { + throw new MetaException(HiveConf.ConfVars.METASTORESCRATCHTRASH.varname + + " is not set in the config or blank"); + } } /** @@ -120,6 +137,45 @@ return whRoot; } + /** + * The same as getWhRoot expect that this returns the scratch trash path, + * which is used for "drop *" operations. + */ + private Path getScratchTrashRootPath() throws MetaException { + if (whScratchTrashPath != null) { + return whScratchTrashPath; + } + whScratchTrashPath = new Path(whScratchString); + return whScratchTrashPath; + } + + public Path getScratchTrashPath(Path originalPath) throws MetaException { + String today = dateFormater.format(calendar.getTime()); + String schema = originalPath.toUri().getScheme(); + String authority = originalPath.toUri().getAuthority(); + FileSystem scratchTrashFs = getFs(originalPath); + + String pathStr = getScratchTrashRootPath().toUri().getPath() + File.separator + today + + File.separator; + + String suffix = originalPath.toUri().getPath(); + + Path scratchPath = new Path(schema, authority, pathStr + suffix) ; + try { + boolean exist = scratchTrashFs.exists(scratchPath); + while (exist) { + String newSuffix = rand.nextInt() + File.separator + suffix; + scratchPath = new Path(schema, authority, pathStr + newSuffix) ; + exist = scratchTrashFs.exists(scratchPath); + } + } catch (IOException e) { + closeFs(scratchTrashFs); + MetaStoreUtils.logAndThrowMetaException(e); + } + + return scratchPath; + } + public Path getDefaultDatabasePath(String dbName) throws MetaException { if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) { return getWhRoot(); @@ -341,4 +397,20 @@ return values; } + public boolean mvDirIfSrcExists(Path tblPath, Path scratchTrash) throws MetaException { + FileSystem fs = null; + try { + fs = getFs(tblPath); + if (fs.exists(tblPath)) { + return fs.rename(tblPath, scratchTrash); + } else { + return true; + } + } catch (Exception e) { + closeFs(fs); + MetaStoreUtils.logAndThrowMetaException(e); + return false; + } + } + } Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java =================================================================== --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1001862) +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy) @@ -206,6 +206,9 @@ conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, (new Path(dfs.getFileSystem().getUri().toString(), "/build/ql/test/data/warehouse/")).toString()); + conf.setVar(HiveConf.ConfVars.METASTORESCRATCHTRASH, + (new Path(dfs.getFileSystem().getUri().toString(), + "/build/ql/test/data/warehouse_scratch/")).toString()); conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort()); } }