diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 3004f9c..a05f893 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql; +import java.io.Closeable; import java.io.DataInput; import java.io.FileNotFoundException; import java.io.IOException; @@ -64,27 +65,29 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.wm.WmContext; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Context for Semantic Analyzers. Usage: not reusable - construct a new one for - * each query should call clear() at end of use to remove temporary folders + * Context for Semantic Analyzers. Usage: not reusable. Construct a new one for + * each query. Call {@link #close()} at end of each use to remove temporary + * folders and to free internal resources */ -public class Context { +public class Context implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(Context.class); + private boolean isHDFSCleanup; private Path resFile; private Path resDir; private FileSystem resFs; - private static final Logger LOG = LoggerFactory.getLogger("hive.ql.Context"); private Path[] resDirPaths; private int resDirFilesNum; boolean initialized; String originalTracker = null; private CompilationOpContext opContext; - private final Map pathToCS = new ConcurrentHashMap(); + private Map pathToCS = new ConcurrentHashMap(); // scratch path to use for all non-local (ie. hdfs) file system tmp folders private final Path nonLocalScratchPath; @@ -99,7 +102,7 @@ private final Map fsScratchDirs = new HashMap(); private Configuration conf; - protected int pathid = 10000; + protected AtomicInteger pathid = new AtomicInteger(10000); protected ExplainConfiguration explainConfig = null; protected String cboInfo; protected boolean cboSucceeded; @@ -617,12 +620,11 @@ public void removeScratchDir() { try { Path p = entry.getValue(); FileSystem fs = p.getFileSystem(conf); - LOG.debug("Deleting scratch dir: {}", p); + LOG.debug("Deleting scratch dir: {}", p); fs.delete(p, true); fs.cancelDeleteOnExit(p); } catch (Exception e) { - LOG.warn("Error Removing Scratch: " - + StringUtils.stringifyException(e)); + LOG.warn("Error Removing Scratch", e); } } fsScratchDirs.clear(); @@ -642,18 +644,17 @@ public void removeMaterializedCTEs() { + materializedTable.getTableName() + ", status=" + status); } catch (IOException e) { // ignore - LOG.warn("Error removing " + location + " for materialized " + materializedTable.getTableName() + - ": " + StringUtils.stringifyException(e)); + LOG.warn("Error removing " + location + " for materialized " + + materializedTable.getTableName(), e); } } cteTables.clear(); } private String nextPathId() { - return Integer.toString(pathid++); + return Integer.toString(pathid.getAndIncrement()); } - private static final String MR_PREFIX = "-mr-"; public static final String EXT_PREFIX = "-ext-"; private static final String LOCAL_PREFIX = "-local-"; @@ -755,29 +756,34 @@ public void setResDir(Path resDir) { resDirPaths = null; } - public void clear() throws IOException { + @Override + public void close() throws IOException { + // Let garbage collection happen + pathToCS = null; + resDirPaths = null; + // First clear the other contexts created by this query for (Context subContext : rewrittenStatementContexts) { - subContext.clear(); + subContext.close(); } // Then clear this context if (resDir != null) { try { FileSystem fs = resDir.getFileSystem(conf); - LOG.debug("Deleting result dir: {}", resDir); + LOG.debug("Deleting result dir: {}", resDir); fs.delete(resDir, true); } catch (IOException e) { - LOG.info("Context clear error: " + StringUtils.stringifyException(e)); + LOG.warn("Context clear error", e); } } if (resFile != null) { try { FileSystem fs = resFile.getFileSystem(conf); - LOG.debug("Deleting result file: {}", resFile); + LOG.debug("Deleting result file: {}", resFile); fs.delete(resFile, false); } catch (IOException e) { - LOG.info("Context clear error: " + StringUtils.stringifyException(e)); + LOG.warn("Context clear error", e); } } removeMaterializedCTEs(); @@ -786,6 +792,11 @@ public void clear() throws IOException { setNeedLockMgr(false); } + @Deprecated + public void clear() throws IOException { + close(); + } + public DataInput getStream() { try { if (!initialized) { @@ -818,10 +829,10 @@ public DataInput getStream() { return getNextStream(); } } catch (FileNotFoundException e) { - LOG.info("getStream error: " + StringUtils.stringifyException(e)); + LOG.info("getStream error", e); return null; } catch (IOException e) { - LOG.info("getStream error: " + StringUtils.stringifyException(e)); + LOG.info("getStream error", e); return null; } } @@ -833,10 +844,10 @@ private DataInput getNextStream() { return resFs.open(resDirPaths[resDirFilesNum++]); } } catch (FileNotFoundException e) { - LOG.info("getNextStream error: " + StringUtils.stringifyException(e)); + LOG.info("getNextStream error", e); return null; } catch (IOException e) { - LOG.info("getNextStream error: " + StringUtils.stringifyException(e)); + LOG.info("getNextStream error", e); return null; } @@ -850,12 +861,6 @@ public void resetStream() { } } - /** - * Little abbreviation for StringUtils. - */ - private static boolean strEquals(String str1, String str2) { - return org.apache.commons.lang.StringUtils.equals(str1, str2); - } /** * Set the token rewrite stream being used to parse the current top-level SQL @@ -1133,5 +1138,4 @@ public Table getTempTableForLoad() { public void setTempTableForLoad(Table tempTableForLoad) { this.tempTableForLoad = tempTableForLoad; } - }