diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java new file mode 100644 index 0000000..93c15c0 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.exec; + +/** + * Interface for common functionality between FunctionInfo/WindowFunctionInfo + */ +public interface CommonFunctionInfo { + Class getFunctionClass(); +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java index 053edf5..da2b3c7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java @@ -30,7 +30,7 @@ * FunctionInfo. * */ -public class FunctionInfo { +public class FunctionInfo implements CommonFunctionInfo { private final boolean isNative; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 6045268..b7da95e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -575,7 +575,8 @@ private static FunctionInfo getFunctionInfoFromMetastore(String functionName) { return ret; } - private static T getQualifiedFunctionInfo(Map mFunctions, String functionName) { + private static T getQualifiedFunctionInfo( + Map mFunctions, String functionName) { T functionInfo = mFunctions.get(functionName); if (functionInfo == null) { // Try looking up in metastore. @@ -585,10 +586,55 @@ private static FunctionInfo getFunctionInfoFromMetastore(String functionName) { functionInfo = mFunctions.get(functionName); } } + + // HIVE-6672: In HiveServer2 the JARs for this UDF may have been loaded by a different thread, + // and the current thread may not be able to resolve the UDF. Test for this condition + // and if necessary load the JARs in this thread. + if (functionInfo != null) { + loadFunctionResourcesIfNecessary(functionName, functionInfo); + } + return functionInfo; } - private static T getFunctionInfo(Map mFunctions, String functionName) { + private static void checkFunctionClass(CommonFunctionInfo cfi) throws ClassNotFoundException { + // This call will fail for non-generic UDFs using GenericUDFBridge + Class udfClass = cfi.getFunctionClass(); + // Even if we have a reference to the class (which will be the case for GenericUDFs), + // the classloader may not be able to resolve the class, which would mean reflection-based + // methods would fail such as for plan deserialization. Make sure this works too. + Class.forName(udfClass.getName(), true, JavaUtils.getClassLoader()); + } + + private static void loadFunctionResourcesIfNecessary(String functionName, CommonFunctionInfo cfi) { + try { + // Check if the necessary JARs have been loaded for this function. + checkFunctionClass(cfi); + } catch (Exception e) { + // Unable to resolve the UDF with the classloader. + // Look up the function in the metastore and load any resources. + LOG.debug("Attempting to reload resources for " + functionName); + try { + String[] parts = FunctionUtils.getQualifiedFunctionNameParts(functionName); + HiveConf conf = SessionState.get().getConf(); + Function func = Hive.get(conf).getFunction(parts[0], parts[1]); + if (func != null) { + FunctionTask.addFunctionResources(func.getResourceUris()); + // Check again now that we've loaded the resources in this thread. + checkFunctionClass(cfi); + } else { + // Couldn't find the function .. just rethrow the original error + LOG.error("Unable to reload resources for " + functionName); + throw e; + } + } catch (Exception err) { + throw new RuntimeException(err); + } + } + } + + private static T getFunctionInfo( + Map mFunctions, String functionName) { functionName = functionName.toLowerCase(); T functionInfo = null; if (FunctionUtils.isQualifiedFunctionName(functionName)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java index f136500..37139a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; @SuppressWarnings("deprecation") -public class WindowFunctionInfo +public class WindowFunctionInfo implements CommonFunctionInfo { boolean supportsWindow = true; boolean pivotResult = false; @@ -59,4 +59,9 @@ public FunctionInfo getfInfo() { return fInfo; } + + @Override + public Class getFunctionClass() { + return getfInfo().getFunctionClass(); + } }