diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 482bd79..eea1e7c 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -770,6 +770,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true, "Whether Hive should use memory-optimized hash table for MapJoin.\n" + "Only works on Tez and Spark, because memory-optimized hashtable cannot be serialized."), + HIVEMAPJOINOPTIMIZEDTABLEPROBEPERCENT("hive.mapjoin.optimized.hashtable.probe.percent", + (float) 0.5, "Probing space percentage of the optimized hashtable"), HIVEUSEHYBRIDGRACEHASHJOIN("hive.mapjoin.hybridgrace.hashtable", true, "Whether to use hybrid" + "grace hash join as the join method for mapjoin. Tez only."), HIVEHYBRIDGRACEHASHJOINMEMCHECKFREQ("hive.mapjoin.hybridgrace.memcheckfrequency", 1024, "For " + diff --git ql/src/java/org/apache/hadoop/hive/ql/Context.java ql/src/java/org/apache/hadoop/hive/ql/Context.java index 1499a91..a92331a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; -import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObj; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; @@ -348,8 +347,7 @@ public void removeScratchDir() { try { Path p = entry.getValue(); FileSystem fs = p.getFileSystem(conf); - fs.delete(p, true); - fs.cancelDeleteOnExit(p); + ShimLoader.getHadoopShims().removeAndCancelDeleteOnExit(fs, p); } catch (Exception e) { LOG.warn("Error Removing Scratch: " + StringUtils.stringifyException(e)); diff --git shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index ffffcb7..bf39928 100644 --- shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@ -722,4 +722,10 @@ public void addDelegationTokens(FileSystem fs, Credentials cred, String uname) t Token fsToken = fs.getDelegationToken(uname); cred.addToken(fsToken.getService(), fsToken); } + + @Override + public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException { + fs.delete(path, true); + // fs.cancelDeleteOnExit() is not available until 0.23.3 + } } diff --git shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 443153a..7756574 100644 --- shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -1405,4 +1405,10 @@ public TextReaderShim getTextReaderShim(InputStream in) throws IOException { } return new FastTextReaderShim(in); } + + @Override + public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException { + fs.delete(path, true); + fs.cancelDeleteOnExit(path); + } } diff --git shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java index 5c6a4ca..74b7114 100644 --- shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java +++ shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java @@ -753,4 +753,9 @@ public void deleteKey(String keyName) throws IOException { * which are required for TextReaderShim.read() input. */ public TextReaderShim getTextReaderShim(InputStream input) throws IOException; + + /** + * Delete a specified path and cancel its delete on exit when such method is available. + */ + public void removeAndCancelDeleteOnExit(FileSystem fs, Path path) throws IOException; }