diff --git beeline/src/java/org/apache/hive/beeline/Commands.java beeline/src/java/org/apache/hive/beeline/Commands.java index f4dd586e11..90cae9f408 100644 --- beeline/src/java/org/apache/hive/beeline/Commands.java +++ beeline/src/java/org/apache/hive/beeline/Commands.java @@ -169,7 +169,7 @@ public boolean addlocaldriverjar(String line) { return false; } - URLClassLoader classLoader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); try { beeLine.debug(jarPath + " is added to the local beeline."); URLClassLoader newClassLoader = new URLClassLoader(new URL[]{p.toURL()}, classLoader); diff --git common/src/java/org/apache/hadoop/hive/common/JavaUtils.java common/src/java/org/apache/hadoop/hive/common/JavaUtils.java index c011cd1626..19066196aa 100644 --- common/src/java/org/apache/hadoop/hive/common/JavaUtils.java +++ common/src/java/org/apache/hadoop/hive/common/JavaUtils.java @@ -91,8 +91,10 @@ public static boolean closeClassLoadersTo(ClassLoader current, ClassLoader stop) try { closeClassLoader(current); } catch (IOException e) { - LOG.info("Failed to close class loader " + current + - Arrays.toString(((URLClassLoader) current).getURLs()), e); + String detailedMessage = current instanceof URLClassLoader ? + Arrays.toString(((URLClassLoader) current).getURLs()) : + ""; + LOG.info("Failed to close class loader " + current + " " + detailedMessage, e); } } return true; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java index 2a6ef3a246..7925b8fdc8 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/FunctionLocalizer.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URLClassLoader; import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.LinkedList; @@ -36,6 +35,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.FunctionTask; +import org.apache.hadoop.hive.ql.exec.UDFClassLoader; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -60,7 +60,7 @@ private final Thread workThread; private final File localDir; private final Configuration conf; - private final URLClassLoader executorClassloader; + private final UDFClassLoader executorClassloader; private final IdentityHashMap, Boolean> allowedUdfClasses = new IdentityHashMap<>(); @@ -70,8 +70,8 @@ public FunctionLocalizer(Configuration conf, String localDir) { this.conf = conf; this.localDir = new File(localDir, DIR_NAME); - this.executorClassloader = (URLClassLoader)Utilities.createUDFClassLoader( - (URLClassLoader)Thread.currentThread().getContextClassLoader(), new String[]{}); + this.executorClassloader = Utilities.createUDFClassLoader( + Thread.currentThread().getContextClassLoader(), new String[]{}); this.workThread = new Thread(new Runnable() { @Override public void run() { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 36bc08f34e..2a35145116 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -68,6 +68,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; @@ -2110,28 +2111,23 @@ private static boolean useExistingClassLoader(ClassLoader cl) { * Array of classpath elements */ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) { - final URLClassLoader loader = (URLClassLoader) cloader; if (useExistingClassLoader(cloader)) { - final UDFClassLoader udfClassLoader = (UDFClassLoader) loader; + final UDFClassLoader udfClassLoader = (UDFClassLoader) cloader; for (String path : newPaths) { udfClassLoader.addURL(urlFromPathString(path)); } return udfClassLoader; } else { - return createUDFClassLoader(loader, newPaths); + return createUDFClassLoader(cloader, newPaths); } } - public static ClassLoader createUDFClassLoader(URLClassLoader loader, String[] newPaths) { - final Set curPathsSet = Sets.newHashSet(loader.getURLs()); - final List curPaths = Lists.newArrayList(curPathsSet); - for (String onestr : newPaths) { - final URL oneurl = urlFromPathString(onestr); - if (oneurl != null && !curPathsSet.contains(oneurl)) { - curPaths.add(oneurl); - } - } - return new UDFClassLoader(curPaths.toArray(new URL[0]), loader); + public static UDFClassLoader createUDFClassLoader(ClassLoader loader, String[] newPaths) { + return new UDFClassLoader(Arrays.stream(newPaths) + .map(Utilities::urlFromPathString) + .filter(url -> url != null) + .collect(Collectors.toList()) + .toArray(new URL[0]), loader); } /** @@ -2142,7 +2138,15 @@ public static ClassLoader createUDFClassLoader(URLClassLoader loader, String[] n */ public static void removeFromClassPath(String[] pathsToRemove) throws IOException { Thread curThread = Thread.currentThread(); - URLClassLoader loader = (URLClassLoader) curThread.getContextClassLoader(); + if (!(curThread.getContextClassLoader() instanceof UDFClassLoader)) { + // Current class loader is + // * either system class loader -- we shouldn't close it! + // * or UDFClassLoader -- then we can remove by closing and creating a more limited one + return; + } + + UDFClassLoader loader = (UDFClassLoader) curThread.getContextClassLoader(); + Set newPath = new HashSet(Arrays.asList(loader.getURLs())); for (String onestr : pathsToRemove) { @@ -2152,9 +2156,9 @@ public static void removeFromClassPath(String[] pathsToRemove) throws IOExceptio } } JavaUtils.closeClassLoader(loader); - // This loader is closed, remove it from cached registry loaders to avoid removing it again. + // This loader is closed, remove it from cached registry loaders to avoid removing it again. Registry reg = SessionState.getRegistry(); - if(reg != null) { + if (reg != null) { reg.removeFromUDFLoaders(loader); } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java index 91868a4667..10eeba8f98 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapper.java @@ -73,20 +73,22 @@ private MapredLocalWork localWork = null; private ExecMapperContext execContext = null; + private static String tryGetClassPath(ClassLoader loader) { + if(loader instanceof URLClassLoader) { + return Arrays.asList(((URLClassLoader) loader).getURLs()).toString(); + } else { + return "unavailable for " + loader.getClass().getSimpleName(); + } + } + @Override public void configure(JobConf job) { execContext = new ExecMapperContext(job); // Allocate the bean at the beginning - - try { - l4j.info("conf classpath = " - + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs())); - l4j.info("thread classpath = " - + Arrays.asList(((URLClassLoader) Thread.currentThread() - .getContextClassLoader()).getURLs())); - } catch (Exception e) { - l4j.info("cannot get classpath: " + e.getMessage()); + if (l4j.isInfoEnabled()) { + l4j.info("conf classpath = " + tryGetClassPath(job.getClassLoader())); + l4j.info("thread classpath = " + tryGetClassPath(Thread.currentThread().getContextClassLoader())); } - setDone(false); try { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java index e106bc9149..1bd2cc73dc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecReducer.java @@ -88,6 +88,14 @@ private transient Object keyObject; private transient BytesWritable groupKey; + private static String tryGetClassPath(ClassLoader loader) { + if(loader instanceof URLClassLoader) { + return Arrays.asList(((URLClassLoader) loader).getURLs()).toString(); + } else { + return "unavailable for " + loader.getClass().getSimpleName(); + } + } + @Override public void configure(JobConf job) { rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE]; @@ -95,15 +103,8 @@ public void configure(JobConf job) { ObjectInspector keyObjectInspector; if (LOG.isInfoEnabled()) { - try { - LOG.info("conf classpath = " - + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs())); - LOG.info("thread classpath = " - + Arrays.asList(((URLClassLoader) Thread.currentThread() - .getContextClassLoader()).getURLs())); - } catch (Exception e) { - LOG.info("cannot get classpath: " + e.getMessage()); - } + LOG.info("conf classpath = " + tryGetClassPath(job.getClassLoader())); + LOG.info("thread classpath = " + tryGetClassPath(Thread.currentThread().getContextClassLoader())); } jc = job; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java index f7ea212cfb..2672be6e90 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkRecordHandler.java @@ -78,6 +78,14 @@ private ScheduledThreadPoolExecutor getMemoryAndRowLogExecutor() { return executor; } + private static String tryGetClassPath(ClassLoader loader) { + if(loader instanceof URLClassLoader) { + return Arrays.asList(((URLClassLoader) loader).getURLs()).toString(); + } else { + return "unavailable for " + loader.getClass().getSimpleName(); + } + } + public void init(JobConf job, OutputCollector output, Reporter reporter) throws Exception { jc = job; MapredContext.init(false, new JobConf(jc)); @@ -89,13 +97,8 @@ private ScheduledThreadPoolExecutor getMemoryAndRowLogExecutor() { LOG.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax()); MemoryInfoLogger memoryInfoLogger = new MemoryInfoLogger(); memoryInfoLogger.run(); - try { - LOG.info("conf classpath = " + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs())); - LOG.info("thread classpath = " + Arrays - .asList(((URLClassLoader) Thread.currentThread().getContextClassLoader()).getURLs())); - } catch (Exception e) { - LOG.info("cannot get classpath: " + e.getMessage()); - } + LOG.info("conf classpath = " + tryGetClassPath(job.getClassLoader())); + LOG.info("thread classpath = " + tryGetClassPath(Thread.currentThread().getContextClassLoader())); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java index 0ec7a04ce7..90454b81ad 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java @@ -66,6 +66,14 @@ public RecordProcessor(JobConf jConf, ProcessorContext processorContext) { this.processorContext = processorContext; } + private static String tryGetClassPath(ClassLoader loader) { + if(loader instanceof URLClassLoader) { + return Arrays.asList(((URLClassLoader) loader).getURLs()).toString(); + } else { + return "unavailable for " + loader.getClass().getSimpleName(); + } + } + /** * Common initialization code for RecordProcessors * @param mrReporter @@ -82,16 +90,9 @@ void init(MRTaskReporter mrReporter, checkAbortCondition(); //log classpaths - try { - if (l4j.isDebugEnabled()) { - l4j.debug("conf classpath = " - + Arrays.asList(((URLClassLoader) jconf.getClassLoader()).getURLs())); - l4j.debug("thread classpath = " - + Arrays.asList(((URLClassLoader) Thread.currentThread() - .getContextClassLoader()).getURLs())); - } - } catch (Exception e) { - l4j.info("cannot get classpath: " + e.getMessage()); + if (l4j.isDebugEnabled()) { + l4j.debug("conf classpath = " + tryGetClassPath(jconf.getClassLoader())); + l4j.debug("thread classpath = " + tryGetClassPath(Thread.currentThread().getContextClassLoader())); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index de5cd8b992..8331d87f05 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -423,7 +423,7 @@ public SessionState(HiveConf conf, String userName) { // classloader as parent can pollute the session. See HIVE-11878 parentLoader = SessionState.class.getClassLoader(); // Make sure that each session has its own UDFClassloader. For details see {@link UDFClassLoader} - final ClassLoader currentLoader = Utilities.createUDFClassLoader((URLClassLoader) parentLoader, new String[]{}); + final ClassLoader currentLoader = Utilities.createUDFClassLoader(parentLoader, new String[]{}); this.sessionConf.setClassLoader(currentLoader); resourceDownloader = new ResourceDownloader(conf, HiveConf.getVar(conf, ConfVars.DOWNLOADED_RESOURCES_DIR)); @@ -1326,16 +1326,15 @@ public void loadAuxJars() throws IOException { return; } - URLClassLoader currentCLoader = - (URLClassLoader) SessionState.get().getConf().getClassLoader(); - currentCLoader = - (URLClassLoader) Utilities.addToClassPath(currentCLoader, jarPaths); + ClassLoader currentCLoader = SessionState.get().getConf().getClassLoader(); + currentCLoader = Utilities.addToClassPath(currentCLoader, jarPaths); sessionConf.setClassLoader(currentCLoader); Thread.currentThread().setContextClassLoader(currentCLoader); } /** * Reload the jars under the path specified in hive.reloadable.aux.jars.path property. + * * @throws IOException */ public void loadReloadableAuxJars() throws IOException { @@ -1350,7 +1349,7 @@ public void loadReloadableAuxJars() throws IOException { Set jarPaths = FileUtils.getJarFilesByPath(renewableJarPath, sessionConf); // load jars under the hive.reloadable.aux.jars.path - if(!jarPaths.isEmpty()){ + if (!jarPaths.isEmpty()) { reloadedAuxJars.addAll(jarPaths); } @@ -1360,10 +1359,10 @@ public void loadReloadableAuxJars() throws IOException { } if (reloadedAuxJars != null && !reloadedAuxJars.isEmpty()) { - URLClassLoader currentCLoader = - (URLClassLoader) SessionState.get().getConf().getClassLoader(); + ClassLoader currentCLoader = + SessionState.get().getConf().getClassLoader(); currentCLoader = - (URLClassLoader) Utilities.addToClassPath(currentCLoader, + Utilities.addToClassPath(currentCLoader, reloadedAuxJars.toArray(new String[0])); sessionConf.setClassLoader(currentCLoader); Thread.currentThread().setContextClassLoader(currentCLoader); diff --git spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java index b434d8f7b7..2299394fc1 100644 --- spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java +++ spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java @@ -28,6 +28,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -51,16 +52,34 @@ public static final String HIVE_KRYO_REG_NAME = "org.apache.hive.spark.HiveKryoRegistrator"; private static final String HIVE_KRYO_REG_JAR_NAME = "hive-kryo-registrator"; private static final ImmutableList ERROR_KEYWORDS = ImmutableList.of("error", "exception"); + /** * Add new elements to the classpath. + * Returns currently known class paths as best effort. For system class loader, this may return empty. + * In such cases we will anyway create new child class loader in {@link #addToClassPath(Map, Configuration, File)}, + * so all new class paths will be added and next time we will have a URLClassLoader to work with. + */ + private static List getCurrentClassPaths(ClassLoader parentLoader) { + if(parentLoader instanceof URLClassLoader) { + return Lists.newArrayList(((URLClassLoader) parentLoader).getURLs()); + } else { + return Collections.emptyList(); + } + } + + /** + * Add new elements to the classpath by creating a child ClassLoader containing both the old and the new paths. + * Similar to {@link org.apache.hadoop.hive.ql.exec.Utilities#addToClassPath(java.lang.ClassLoader, java.lang.String[])}. + * This method supports downloading HDFS files to local FS if missing from cache or later timestamp. + * However, this method has no tricks working around HIVE-11878, like UDFClassLoader.... * * @param newPaths Map of classpath elements and corresponding timestamp * @return locally accessible files corresponding to the newPaths */ public static List addToClassPath(Map newPaths, Configuration conf, File localTmpDir) throws Exception { - URLClassLoader loader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); - List curPath = Lists.newArrayList(loader.getURLs()); + ClassLoader parentLoader = Thread.currentThread().getContextClassLoader(); + List curPath = getCurrentClassPaths(parentLoader); List localNewPaths = new ArrayList<>(); boolean newPathAdded = false; @@ -76,7 +95,7 @@ if (newPathAdded) { URLClassLoader newLoader = - new URLClassLoader(curPath.toArray(new URL[curPath.size()]), loader); + new URLClassLoader(curPath.toArray(new URL[curPath.size()]), parentLoader); Thread.currentThread().setContextClassLoader(newLoader); } return localNewPaths; diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 0642b39f58..a3f2d29506 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -25,6 +25,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -40,6 +41,7 @@ import javax.annotation.Nullable; +import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -378,6 +380,19 @@ public static int getArchivingLevel(Partition part) throws MetaException { return HadoopThriftAuthBridge.getBridge().getHadoopSaslProperties(conf); } + /** + * Returns currently known class paths as best effort. For system class loader, this may return + * In such cases we will anyway create new child class loader in {@link #addToClassPath(ClassLo + * so all new class paths will be added and next time we will have a URLClassLoader to work wit + */ + private static List getCurrentClassPaths(ClassLoader parentLoader) { + if(parentLoader instanceof URLClassLoader) { + return Lists.newArrayList(((URLClassLoader) parentLoader).getURLs()); + } else { + return Collections.emptyList(); + } + } + /** * Add new elements to the classpath. * @@ -385,8 +400,7 @@ public static int getArchivingLevel(Partition part) throws MetaException { * Array of classpath elements */ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception { - URLClassLoader loader = (URLClassLoader) cloader; - List curPath = Arrays.asList(loader.getURLs()); + List curPath = getCurrentClassPaths(cloader); ArrayList newPath = new ArrayList<>(curPath.size()); // get a list with the current classpath components @@ -402,7 +416,7 @@ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) } } - return new URLClassLoader(curPath.toArray(new URL[0]), loader); + return new URLClassLoader(curPath.toArray(new URL[0]), cloader); } /**