diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index d618ef96f30..9f59d4cea3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.net.URI; import java.text.SimpleDateFormat; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -671,6 +672,10 @@ public void removeScratchDir() { for (Map.Entry entry : fsScratchDirs.entrySet()) { try { Path p = entry.getValue(); + if (p.toUri().getPath().contains(stagingDir) && subDirOf(p, fsScratchDirs.values()) ) { + LOG.debug("Skip deleting stagingDir: " + p); + continue; // staging dir is deleted when deleting the scratch dir + } if(resultCacheDir == null || !p.toUri().getPath().contains(resultCacheDir)) { // delete only the paths which aren't result cache dir path // because that will be taken care by removeResultCacheDir @@ -687,6 +692,15 @@ public void removeScratchDir() { fsScratchDirs.clear(); } + private boolean subDirOf(Path path, Collection parents) { + for (Path each : parents) { + if (!path.equals(each) && FileUtils.isPathWithinSubtree(path, each)) { + return true; + } + } + return false; + } + /** * Remove any created directories for CTEs. */ @@ -843,7 +857,7 @@ public void clear(boolean deleteResultDir) throws IOException { subContext.clear(); } // Then clear this context - if (resDir != null) { + if (resDir != null && !isInScratchDir(resDir)) { // resDir is inside the scratch dir, removeScratchDir will take care of removing it try { FileSystem fs = resDir.getFileSystem(conf); LOG.debug("Deleting result dir: {}", resDir); @@ -853,7 +867,7 @@ public void clear(boolean deleteResultDir) throws IOException { } } - if (resFile != null) { + if (resFile != null && !isInScratchDir(resFile.getParent())) { // resFile is inside the scratch dir, removeScratchDir will take care of removing it try { FileSystem fs = resFile.getFileSystem(conf); LOG.debug("Deleting result file: {}", resFile); @@ -871,6 +885,11 @@ public void clear(boolean deleteResultDir) throws IOException { setNeedLockMgr(false); } + private boolean isInScratchDir(Path path) { + return path.toUri().getPath().startsWith(localScratchDir) + || path.toUri().getPath().startsWith(nonLocalScratchPath.toUri().getPath()); + } + public DataInput getStream() { try { if (!initialized) {