diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index c1a2a46..30299f2 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -335,7 +335,7 @@ private static Properties getDataSourceProps(Configuration conf) { return prop; } - private static synchronized PersistenceManagerFactory getPMF() { + public static synchronized PersistenceManagerFactory getPMF() { if (pmf == null) { pmf = JDOHelper.getPersistenceManagerFactory(prop); DataStoreCache dsc = pmf.getDataStoreCache(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 5f6f818..245d7a4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; @@ -41,6 +42,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.MapRedStats; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -69,6 +71,9 @@ import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; +import org.datanucleus.ClassLoaderResolver; +import org.datanucleus.NucleusContext; +import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; import com.google.common.base.Preconditions; @@ -1275,10 +1280,29 @@ public void close() throws IOException { + // HIVE-11499 + // DataNucleus caches classloaders in NucleusContext. + // In UDFs, this can result in classloaders not getting GCed resulting in PermGen leaks. + // The NucleusContext cache gets freed up only on calling a close on it. + // We're not closing NucleusContext since it does a bunch of other things which we don't want. + // We're not clearing the cache HashMap by calling HashMap#clear to avoid concurrency issues. + if (ObjectStore.getPMF() instanceof JDOPersistenceManagerFactory) { + JDOPersistenceManagerFactory pmf = (JDOPersistenceManagerFactory) ObjectStore.getPMF(); + NucleusContext nc = pmf.getNucleusContext(); + try { + Field classLoaderResolverMap = NucleusContext.class.getDeclaredField("classLoaderResolverMap"); + classLoaderResolverMap.setAccessible(true); + classLoaderResolverMap.set(nc, new HashMap()); + } catch (Exception e) { + LOG.info(e); + } + LOG.info("Removed cached classloaders from DataNucleus NucleusContext"); + } } public AuthorizationMode getAuthorizationMode(){