diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 3d3dddf..cdcee11 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -788,6 +788,9 @@ public static void deleteDirectory(File directory) throws IOException { /** * create temporary file and register it to delete-on-exit hook. * File.deleteOnExit is not used for possible memory leakage. + * + * Make sure to use {@link #deleteTmpFile(File)} after the file is no longer required, + * and has been deleted to avoid a memory leak. */ public static File createTempFile(String lScratchDir, String prefix, String suffix) throws IOException { File tmpDir = lScratchDir == null ? null : new File(lScratchDir); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 2607db1..c1f7024 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -561,9 +561,8 @@ private Path createTezDir(String sessionId) throws IOException { fs.mkdirs(tezDir, fsPermission); // Make sure the path is normalized (we expect validation to pass since we just created it). tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath(); - // don't keep the directory around on non-clean exit - fs.deleteOnExit(tezDir); + // Directory removal will be handled by cleanup at the SessionState level. return tezDir; } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 0581bab..8d882e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -659,7 +659,9 @@ private void createSessionDirs(String userName) throws IOException { conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString()); // 7. HDFS temp table space hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX); - createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false, true); + // This is a sub-dir under the hdfsSessionPath. Will be removed along with that dir. + // Don't register with deleteOnExit + createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false, false); conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString()); } @@ -782,14 +784,30 @@ void releaseSessionLockFile() throws IOException { private void dropSessionPaths(Configuration conf) throws IOException { if (hdfsSessionPath != null) { if (hdfsSessionPathLockFile != null) { - hdfsSessionPathLockFile.close(); + try { + hdfsSessionPathLockFile.close(); + } catch (IOException e) { + LOG.error("Failed while closing hdfsSessionPathLockFileStream", e); + } + } + FileSystem fs = hdfsSessionPath.getFileSystem(conf); + fs.cancelDeleteOnExit(hdfsSessionPath); + try { + fs.delete(hdfsSessionPath, true); + LOG.info("Deleted HDFS directory: " + hdfsSessionPath); + } catch (IOException e) { + LOG.error("Failed to delete remoteSessionPath at {}", hdfsSessionPath, e); } - hdfsSessionPath.getFileSystem(conf).delete(hdfsSessionPath, true); - LOG.info("Deleted HDFS directory: " + hdfsSessionPath); } if (localSessionPath != null) { - FileSystem.getLocal(conf).delete(localSessionPath, true); - LOG.info("Deleted local directory: " + localSessionPath); + FileSystem fs = FileSystem.getLocal(conf); + fs.cancelDeleteOnExit(localSessionPath); + try { + fs.delete(localSessionPath, true); + LOG.info("Deleted local directory: " + localSessionPath); + } catch (IOException e) { + LOG.error("Failed to delete localSessionPath at {}", localSessionPath, e); + } } deleteTmpOutputFile(); deleteTmpErrOutputFile();