diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java index 2ba91d042cc26bf086e86d8e50859e27cf1c12c4..a5d59ae9bce4bf06e43c36a808e58a1d141bc28d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -20,8 +20,10 @@ import com.google.common.base.Splitter; import com.google.common.collect.Sets; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -43,9 +45,12 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.util.ReflectionUtils; +import java.io.IOException; +import java.net.URLClassLoader; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -66,6 +71,7 @@ */ private final Map mFunctions = new LinkedHashMap(); private final Set> builtIns = Collections.synchronizedSet(new HashSet>()); + private final Set mSessionUDFLoaders = new LinkedHashSet(); private final boolean isNative; @@ -443,7 +449,6 @@ private FunctionInfo getQualifiedFunctionInfo(String qualifiedName) throws Seman // should be called after session registry is checked private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInfo function) { FunctionInfo ret = null; - ClassLoader prev = Utilities.getSessionSpecifiedClassLoader(); try { // Found UDF in metastore - now add it to the function registry @@ -455,7 +460,6 @@ private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInf LOG.error("Unable to load resources for " + qualifiedName + ":" + e, e); return null; } - ClassLoader loader = Utilities.getSessionSpecifiedClassLoader(); Class udfClass = Class.forName(function.getClassName(), true, loader); @@ -463,6 +467,9 @@ private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInf if (ret == null) { LOG.error(function.getClassName() + " is not a valid UDF class and was not registered."); } + if (SessionState.get().isHiveServerQuery()) { + SessionState.getRegistryForWrite().addToUDFLoaders(loader); + } } catch (ClassNotFoundException e) { // Lookup of UDf class failed LOG.error("Unable to load UDF class: " + e); @@ -489,6 +496,24 @@ public synchronized void clear() { builtIns.clear(); } + public synchronized void closeCUDFLoaders() { + try { + for(ClassLoader loader: mSessionUDFLoaders) { + JavaUtils.closeClassLoader(loader); + } + } catch (IOException ie) { + LOG.error("Error in close loader: " + ie); + } + mSessionUDFLoaders.clear(); + } + + public synchronized void addToUDFLoaders(ClassLoader loader) { + mSessionUDFLoaders.add(loader); + } + public synchronized void removeFromUDFLoaders(ClassLoader loader) { + mSessionUDFLoaders.remove(loader); + } + /** * Setup blocked flag for all builtin UDFs as per udf whitelist and blacklist * @param whiteListStr diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 0cfae8bb28c8aa835f976bf9fea34b1da9cc1978..7fed06834e62a33ea47c8719765312f16d67b349 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -2225,6 +2225,11 @@ public static void removeFromClassPath(String[] pathsToRemove) throws Exception } } JavaUtils.closeClassLoader(loader); +//this loader is closed, remove it from cached registry loaders to avoid remove it again. + Registry reg = SessionState.getRegistry(); + if(reg != null) { + reg.removeFromUDFLoaders(loader); + } loader = new URLClassLoader(newPath.toArray(new URL[0])); curThread.setContextClassLoader(loader); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 66cff87d118ee22f2e8308b0b5c70f87d99d5625..0169430208b7666ae096f4066eaaa27f480ada0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -1445,7 +1445,7 @@ public void setCurrentDatabase(String currentDatabase) { } public void close() throws IOException { - registry.clear();; + registry.clear(); if (txnMgr != null) txnMgr.closeTxnManager(); JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader); File resourceDir = @@ -1480,7 +1480,7 @@ public void close() throws IOException { sparkSession = null; } } - + registry.closeCUDFLoaders(); dropSessionPaths(conf); }