diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java index d78cf0c..848be26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java @@ -33,8 +33,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.ql.plan.TezWork.VertexType; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataInputByteBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.serializer.SerializationFactory; import org.apache.hadoop.mapred.FileSplit; diff --git ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java index 9fc99fc..153908c 100755 --- ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java +++ ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.metadata; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_CHECKPOINT_INTERVAL_KEY; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME; import java.util.ArrayList; @@ -28,13 +26,13 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import junit.framework.TestCase; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreUtils; @@ -51,6 +49,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer; import org.apache.hadoop.hive.serde2.thrift.test.Complex; +import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.SequenceFileInputFormat; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.TextInputFormat; @@ -70,8 +69,8 @@ protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); // enable trash so it can be tested - hiveConf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, 30); - hiveConf.setFloat(FS_TRASH_INTERVAL_KEY, 30); + hiveConf.setFloat("fs.trash.checkpoint.interval", 30); // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2) + hiveConf.setFloat("fs.trash.interval", 30); // FS_TRASH_INTERVAL_KEY (hadoop-2) SessionState.start(hiveConf); try { hm = Hive.get(hiveConf); @@ -89,8 +88,8 @@ protected void tearDown() throws Exception { try { super.tearDown(); // disable trash - hiveConf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, 30); - hiveConf.setFloat(FS_TRASH_INTERVAL_KEY, 30); + hiveConf.setFloat("fs.trash.checkpoint.interval", 30); // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2) + hiveConf.setFloat("fs.trash.interval", 30); // FS_TRASH_INTERVAL_KEY (hadoop-2) Hive.closeCurrent(); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); @@ -355,6 +354,9 @@ public void testGetAndDropTables() throws Throwable { } public void testDropTableTrash() throws Throwable { + if (!ShimLoader.getHadoopShims().supportTrashFeature()) { + return; // it's hadoop-1 + } try { String dbName = "db_for_testdroptable"; hm.dropDatabase(dbName, true, true, true); @@ -379,12 +381,9 @@ public void testDropTableTrash() throws Throwable { FileSystem fs = path1.getFileSystem(hiveConf); assertTrue(fs.exists(path1)); // drop table and check that trash works - TrashPolicy tp = TrashPolicy.getInstance(hiveConf, fs, fs.getHomeDirectory()); - assertNotNull("TrashPolicy instance should not be null", tp); - assertTrue("TrashPolicy is not enabled for filesystem: " + fs.getUri(), tp.isEnabled()); - Path trashDir = tp.getCurrentTrashDir(); + Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(hiveConf, fs); assertNotNull("trash directory should not be null", trashDir); - Path trash1 = Path.mergePaths(trashDir, path1); + Path trash1 = mergePaths(trashDir, path1); Path pathglob = trash1.suffix("*");; FileStatus before[] = fs.globStatus(pathglob); hm.dropTable(dbName, ts.get(0)); @@ -399,7 +398,7 @@ public void testDropTableTrash() throws Throwable { assertEquals(ts.get(1), table2.getTableName()); Path path2 = table2.getPath(); assertTrue(fs.exists(path2)); - Path trash2 = Path.mergePaths(trashDir, path2); + Path trash2 = mergePaths(trashDir, path2); System.out.println("trashDir2 is " + trash2); pathglob = trash2.suffix("*"); before = fs.globStatus(pathglob); @@ -621,4 +620,39 @@ public void testHiveRefreshOnConfChange() throws Throwable{ newHiveObj = Hive.get(newHconf); assertTrue(prevHiveObj != newHiveObj); } + + // shamelessly copied from Path in hadoop-2 + private static final String SEPARATOR = "/"; + private static final char SEPARATOR_CHAR = '/'; + + private static final String CUR_DIR = "."; + + private static final boolean WINDOWS + = System.getProperty("os.name").startsWith("Windows"); + + private static final Pattern hasDriveLetterSpecifier = + Pattern.compile("^/?[a-zA-Z]:"); + + private static Path mergePaths(Path path1, Path path2) { + String path2Str = path2.toUri().getPath(); + path2Str = path2Str.substring(startPositionWithoutWindowsDrive(path2Str)); + // Add path components explicitly, because simply concatenating two path + // string is not safe, for example: + // "/" + "/foo" yields "//foo", which will be parsed as authority in Path + return new Path(path1.toUri().getScheme(), + path1.toUri().getAuthority(), + path1.toUri().getPath() + path2Str); + } + + private static int startPositionWithoutWindowsDrive(String path) { + if (hasWindowsDrive(path)) { + return path.charAt(0) == SEPARATOR_CHAR ? 3 : 2; + } else { + return 0; + } + } + + private static boolean hasWindowsDrive(String path) { + return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find()); + } } diff --git shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java index 9592bc4..a353a46 100644 --- shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java +++ shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java @@ -918,4 +918,14 @@ public boolean supportStickyBit() { public boolean hasStickyBit(FsPermission permission) { return false; // not supported } + + @Override + public boolean supportTrashFeature() { + return false; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + return null; + } } diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index bbb3f4e..030cb75 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -534,6 +534,16 @@ public boolean supportStickyBit() { @Override public boolean hasStickyBit(FsPermission permission) { - return false; // not supported + return false; + } + + @Override + public boolean supportTrashFeature() { + return false; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + return null; } } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index afeb073..0731108 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.ProxyFileSystem; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Trash; +import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; @@ -835,4 +836,15 @@ public boolean supportStickyBit() { public boolean hasStickyBit(FsPermission permission) { return permission.getStickyBit(); } + + @Override + public boolean supportTrashFeature() { + return true; + } + + @Override + public Path getCurrentTrashPath(Configuration conf, FileSystem fs) { + TrashPolicy tp = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory()); + return tp.getCurrentTrashDir(); + } } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 282437c..c5f50d2 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -721,4 +721,16 @@ public void checkFileAccess(FileSystem fs, FileStatus status, FsAction action) * @return sticky bit */ boolean hasStickyBit(FsPermission permission); + + /** + * check whether current hadoop supports trash feature + * @return + */ + boolean supportTrashFeature(); + + /** + * returns + * @return + */ + Path getCurrentTrashPath(Configuration conf, FileSystem fs); }