Index: build-common.xml
===================================================================
--- build-common.xml (revision 1001862)
+++ build-common.xml (working copy)
@@ -396,6 +396,7 @@
+
@@ -425,6 +426,7 @@
+
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1001881)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -53,6 +53,7 @@
public static final HiveConf.ConfVars[] metaVars = {
HiveConf.ConfVars.METASTOREDIRECTORY,
HiveConf.ConfVars.METASTOREWAREHOUSE,
+ HiveConf.ConfVars.METASTORESCRATCHTRASH,
HiveConf.ConfVars.METASTOREURIS,
HiveConf.ConfVars.METATORETHRIFTRETRIES,
HiveConf.ConfVars.METASTOREPWD,
@@ -122,6 +123,7 @@
// something here!
METASTOREDIRECTORY("hive.metastore.metadb.dir", ""),
METASTOREWAREHOUSE("hive.metastore.warehouse.dir", ""),
+ METASTORESCRATCHTRASH("hive.metastore.warehouse.scratch.dir", ""),
METASTOREURIS("hive.metastore.uris", ""),
// Number of times to retry a connection to a Thrift metastore server
METATORETHRIFTRETRIES("hive.metastore.connect.retries", 3),
Index: conf/hive-default.xml
===================================================================
--- conf/hive-default.xml (revision 1001881)
+++ conf/hive-default.xml (working copy)
@@ -193,6 +193,12 @@
+ hive.metastore.warehouse.scratch.dir
+ /user/hive/scratch_warehouse
+ location of scratch trash dir for the warehouse
+
+
+
hive.metastore.warehouse.dir
/user/hive/warehouse
location of default database for the warehouse
Index: data/conf/hive-site.xml
===================================================================
--- data/conf/hive-site.xml (revision 1001862)
+++ data/conf/hive-site.xml (working copy)
@@ -65,6 +65,12 @@
+ hive.metastore.warehouse.scratch.dir
+ ${test.warehouse.scratch.dir}
+
+
+
+
hive.metastore.metadb.dir
file://${build.dir}/test/data/metadb/
Index: eclipse-templates/TestCliDriver.launchtemplate
===================================================================
--- eclipse-templates/TestCliDriver.launchtemplate (revision 1001862)
+++ eclipse-templates/TestCliDriver.launchtemplate (working copy)
@@ -21,6 +21,6 @@
-
+
Index: eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate
===================================================================
--- eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate (revision 1001862)
+++ eclipse-templates/TestEmbeddedHiveMetaStore.launchtemplate (working copy)
@@ -21,6 +21,6 @@
-
+
Index: eclipse-templates/TestHive.launchtemplate
===================================================================
--- eclipse-templates/TestHive.launchtemplate (revision 1001862)
+++ eclipse-templates/TestHive.launchtemplate (working copy)
@@ -21,6 +21,6 @@
-
+
Index: eclipse-templates/TestHiveMetaStoreChecker.launchtemplate
===================================================================
--- eclipse-templates/TestHiveMetaStoreChecker.launchtemplate (revision 1001862)
+++ eclipse-templates/TestHiveMetaStoreChecker.launchtemplate (working copy)
@@ -21,6 +21,6 @@
-
+
Index: eclipse-templates/TestJdbc.launchtemplate
===================================================================
--- eclipse-templates/TestJdbc.launchtemplate (revision 1001862)
+++ eclipse-templates/TestJdbc.launchtemplate (working copy)
@@ -21,7 +21,7 @@
-
+
Index: eclipse-templates/TestRemoteHiveMetaStore.launchtemplate
===================================================================
--- eclipse-templates/TestRemoteHiveMetaStore.launchtemplate (revision 1001862)
+++ eclipse-templates/TestRemoteHiveMetaStore.launchtemplate (working copy)
@@ -21,6 +21,6 @@
-
+
Index: metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (revision 1001862)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (working copy)
@@ -850,6 +850,10 @@
Table tbl = null;
isExternal = false;
boolean isIndexTable = false;
+
+ Path scratchTrash = null;
+ boolean movedToTrash = false;
+
try {
ms.openTransaction();
// drop any partitions
@@ -883,6 +887,10 @@
isExternal = isExternal(tbl);
if (tbl.getSd().getLocation() != null) {
tblPath = new Path(tbl.getSd().getLocation());
+ if(deleteData && !isExternal) {
+ scratchTrash = wh.getScratchTrashPath(tblPath);
+ movedToTrash = wh.mvDirIfSrcExists(tblPath, scratchTrash);
+ }
}
if (!ms.dropTable(dbname, name)) {
@@ -893,9 +901,17 @@
} finally {
if (!success) {
ms.rollbackTransaction();
- } else if (deleteData && (tblPath != null) && !isExternal) {
- wh.deleteDir(tblPath, true);
+ if (movedToTrash) {
+ //recover data from hdfs
+ if (!wh.mvDirIfSrcExists(scratchTrash, tblPath)) {
+ throw new MetaException(
+ "Drop table failed, and data in Scratch Trash ("
+ + scratchTrash.toString() + ") can not be recovered!");
+ }
+ }
+ } else if (movedToTrash) {
// ok even if the data is not deleted
+ wh.deleteDir(scratchTrash, true);
}
}
}
@@ -1222,7 +1238,9 @@
Table tbl = null;
Partition part = null;
boolean isArchived = false;
- Path archiveParentDir = null;
+
+ Path scratchTrash = null;
+ boolean movedToTrash = false;
try {
ms.openTransaction();
@@ -1234,9 +1252,6 @@
}
isArchived = MetaStoreUtils.isArchived(part);
- if (isArchived) {
- archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
- }
if (part.getSd() == null || part.getSd().getLocation() == null) {
throw new MetaException("Partition metadata is corrupted");
}
@@ -1245,24 +1260,42 @@
}
success = ms.commitTransaction();
partPath = new Path(part.getSd().getLocation());
+ if (isArchived) {
+ // Archived partitions have har:/to_har_file as their location.
+ // The original directory was saved in params
+ partPath = MetaStoreUtils.getOriginalLocation(part);
+ }
+
tbl = get_table(db_name, tbl_name);
+
+ if (tbl == null) {
+ throw new MetaException("Unable to load table " + tbl_name
+ + " from metastore.");
+ }
+
+ if (partPath != null) {
+ if(deleteData && !isExternal(tbl)) {
+ scratchTrash = wh.getScratchTrashPath(partPath);
+ movedToTrash = wh.mvDirIfSrcExists(partPath, scratchTrash);
+ }
+ }
+
} finally {
if (!success) {
ms.rollbackTransaction();
- } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) {
- if (tbl != null && !isExternal(tbl)) {
- // Archived partitions have har:/to_har_file as their location.
- // The original directory was saved in params
- if (isArchived) {
- assert(archiveParentDir != null);
- wh.deleteDir(archiveParentDir, true);
- } else {
- assert(partPath != null);
- wh.deleteDir(partPath, true);
+ if (movedToTrash) {
+ //recover data from hdfs
+ if (!wh.mvDirIfSrcExists(scratchTrash, partPath)) {
+ throw new MetaException(
+ "Drop partition failed, and data in Scratch Trash ("
+ + scratchTrash.toString() + ") can not be recovered!");
}
- // ok even if the data is not deleted
}
+ } else if (movedToTrash) {
+ // ok even if the data is not deleted
+ wh.deleteDir(scratchTrash, true);
}
+
}
return true;
}
Index: metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (revision 1001862)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (working copy)
@@ -20,12 +20,16 @@
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.GregorianCalendar;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -51,7 +55,15 @@
private final Configuration conf;
private final String whRootString;
+ private final String whScratchString;
+ private Path whScratchTrashPath;
+
private static final String DATABASE_WAREHOUSE_SUFFIX = ".db";
+
+ private GregorianCalendar calendar = new GregorianCalendar();
+ private SimpleDateFormat dateFormater = new SimpleDateFormat("yyyy-MM-dd");
+ private Random rand = new Random();
+
public static final Log LOG = LogFactory.getLog("hive.metastore.warehouse");
@@ -62,6 +74,11 @@
throw new MetaException(HiveConf.ConfVars.METASTOREWAREHOUSE.varname
+ " is not set in the config or blank");
}
+ whScratchString = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORESCRATCHTRASH);
+ if (StringUtils.isBlank(whScratchString)) {
+ throw new MetaException(HiveConf.ConfVars.METASTORESCRATCHTRASH.varname
+ + " is not set in the config or blank");
+ }
}
/**
@@ -120,6 +137,45 @@
return whRoot;
}
+ /**
+ * The same as getWhRoot expect that this returns the scratch trash path,
+ * which is used for "drop *" operations.
+ */
+ private Path getScratchTrashRootPath() throws MetaException {
+ if (whScratchTrashPath != null) {
+ return whScratchTrashPath;
+ }
+ whScratchTrashPath = new Path(whScratchString);
+ return whScratchTrashPath;
+ }
+
+ public Path getScratchTrashPath(Path originalPath) throws MetaException {
+ String today = dateFormater.format(calendar.getTime());
+ String schema = originalPath.toUri().getScheme();
+ String authority = originalPath.toUri().getAuthority();
+ FileSystem scratchTrashFs = getFs(originalPath);
+
+ String pathStr = getScratchTrashRootPath().toUri().getPath() + File.separator + today
+ + File.separator;
+
+ String suffix = originalPath.toUri().getPath();
+
+ Path scratchPath = new Path(schema, authority, pathStr + suffix) ;
+ try {
+ boolean exist = scratchTrashFs.exists(scratchPath);
+ while (exist) {
+ String newSuffix = rand.nextInt() + File.separator + suffix;
+ scratchPath = new Path(schema, authority, pathStr + newSuffix) ;
+ exist = scratchTrashFs.exists(scratchPath);
+ }
+ } catch (IOException e) {
+ closeFs(scratchTrashFs);
+ MetaStoreUtils.logAndThrowMetaException(e);
+ }
+
+ return scratchPath;
+ }
+
public Path getDefaultDatabasePath(String dbName) throws MetaException {
if (dbName.equalsIgnoreCase(DEFAULT_DATABASE_NAME)) {
return getWhRoot();
@@ -341,4 +397,20 @@
return values;
}
+ public boolean mvDirIfSrcExists(Path tblPath, Path scratchTrash) throws MetaException {
+ FileSystem fs = null;
+ try {
+ fs = getFs(tblPath);
+ if (fs.exists(tblPath)) {
+ return fs.rename(tblPath, scratchTrash);
+ } else {
+ return true;
+ }
+ } catch (Exception e) {
+ closeFs(fs);
+ MetaStoreUtils.logAndThrowMetaException(e);
+ return false;
+ }
+ }
+
}
Index: ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
===================================================================
--- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (revision 1001862)
+++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (working copy)
@@ -206,6 +206,9 @@
conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
(new Path(dfs.getFileSystem().getUri().toString(),
"/build/ql/test/data/warehouse/")).toString());
+ conf.setVar(HiveConf.ConfVars.METASTORESCRATCHTRASH,
+ (new Path(dfs.getFileSystem().getUri().toString(),
+ "/build/ql/test/data/warehouse_scratch/")).toString());
conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort());
}
}