diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java index 30ba996..8014dab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java @@ -73,36 +73,36 @@ public FunctionInfo(String displayName, String className, FunctionResource... re this.discarded = new AtomicBoolean(false); // shared to all session functions } - public FunctionInfo(boolean isNative, String displayName, + public FunctionInfo(FunctionType functionType, String displayName, GenericUDF genericUDF, FunctionResource... resources) { - this.functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + this.functionType = functionType; this.displayName = displayName; this.genericUDF = genericUDF; this.isInternalTableFunction = false; this.resources = resources; } - public FunctionInfo(boolean isNative, String displayName, + public FunctionInfo(FunctionType functionType, String displayName, GenericUDAFResolver genericUDAFResolver, FunctionResource... resources) { - this.functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + this.functionType = functionType; this.displayName = displayName; this.genericUDAFResolver = genericUDAFResolver; this.isInternalTableFunction = false; this.resources = resources; } - public FunctionInfo(boolean isNative, String displayName, + public FunctionInfo(FunctionType functionType, String displayName, GenericUDTF genericUDTF, FunctionResource... resources) { - this.functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + this.functionType = functionType; this.displayName = displayName; this.genericUDTF = genericUDTF; this.isInternalTableFunction = false; this.resources = resources; } - public FunctionInfo(boolean isNative, String displayName, Class tFnCls, + public FunctionInfo(FunctionType functionType, String displayName, Class tFnCls, FunctionResource... resources) { - this.functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + this.functionType = functionType; this.displayName = displayName; this.tableFunctionResolver = tFnCls; PartitionTableFunctionDescription def = AnnotationUtils.getAnnotation( @@ -263,6 +263,10 @@ public void shareStateWith(FunctionInfo function) { } } + public FunctionType getFunctionType() { + return functionType; + } + public static class FunctionResource { private final SessionState.ResourceType resourceType; private final String resourceURI; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java index 05926b5..6cca8b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; +import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionType; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -104,31 +105,37 @@ public Registry(boolean isNative) { * @return true if udfClass's type was recognized (so registration * succeeded); false otherwise */ - @SuppressWarnings("unchecked") public FunctionInfo registerFunction( String functionName, Class udfClass, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerFunction(functionName, functionType, udfClass, resources); + } + + @SuppressWarnings("unchecked") + private FunctionInfo registerFunction( + String functionName, FunctionType functionType, Class udfClass, FunctionResource... resources) { FunctionUtils.UDFClassType udfClassType = FunctionUtils.getUDFClassType(udfClass); switch (udfClassType) { case UDF: return registerUDF( - functionName, (Class) udfClass, false, resources); + functionName, functionType, (Class) udfClass, false, functionName.toLowerCase(), resources); case GENERIC_UDF: return registerGenericUDF( - functionName, (Class) udfClass, resources); + functionName, functionType, (Class) udfClass, resources); case GENERIC_UDTF: return registerGenericUDTF( - functionName, (Class) udfClass, resources); + functionName, functionType, (Class) udfClass, resources); case UDAF: return registerUDAF( - functionName, (Class) udfClass, resources); + functionName, functionType, (Class) udfClass, resources); case GENERIC_UDAF_RESOLVER: return registerGenericUDAF( - functionName, (GenericUDAFResolver) - ReflectionUtil.newInstance(udfClass, null), resources); + functionName, functionType, + (GenericUDAFResolver) ReflectionUtil.newInstance(udfClass, null), resources); case TABLE_FUNCTION_RESOLVER: // native or not would be decided by annotation. need to evaluate that first - return registerTableFunction(functionName, + return registerTableFunction(functionName, functionType, (Class) udfClass, resources); } return null; @@ -143,8 +150,15 @@ public FunctionInfo registerUDF(String functionName, public FunctionInfo registerUDF(String functionName, Class UDFClass, boolean isOperator, String displayName, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerUDF(functionName, functionType, UDFClass, isOperator, displayName); + } + + private FunctionInfo registerUDF(String functionName, FunctionType functionType, + Class UDFClass, boolean isOperator, String displayName, + FunctionResource... resources) { validateClass(UDFClass, UDF.class); - FunctionInfo fI = new FunctionInfo(isNative, displayName, + FunctionInfo fI = new FunctionInfo(functionType, displayName, new GenericUDFBridge(displayName, isOperator, UDFClass.getName()), resources); addFunction(functionName, fI); return fI; @@ -152,8 +166,14 @@ public FunctionInfo registerUDF(String functionName, public FunctionInfo registerGenericUDF(String functionName, Class genericUDFClass, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerGenericUDF(functionName, functionType, genericUDFClass, resources); + } + + private FunctionInfo registerGenericUDF(String functionName, FunctionType functionType, + Class genericUDFClass, FunctionResource... resources) { validateClass(genericUDFClass, GenericUDF.class); - FunctionInfo fI = new FunctionInfo(isNative, functionName, + FunctionInfo fI = new FunctionInfo(functionType, functionName, ReflectionUtil.newInstance(genericUDFClass, null), resources); addFunction(functionName, fI); return fI; @@ -177,8 +197,14 @@ public void registerHiddenBuiltIn(Class functionClass) { public FunctionInfo registerGenericUDTF(String functionName, Class genericUDTFClass, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerGenericUDTF(functionName, functionType, genericUDTFClass, resources); + } + + private FunctionInfo registerGenericUDTF(String functionName, FunctionType functionType, + Class genericUDTFClass, FunctionResource... resources) { validateClass(genericUDTFClass, GenericUDTF.class); - FunctionInfo fI = new FunctionInfo(isNative, functionName, + FunctionInfo fI = new FunctionInfo(functionType, functionName, ReflectionUtil.newInstance(genericUDTFClass, null), resources); addFunction(functionName, fI); return fI; @@ -186,8 +212,14 @@ public FunctionInfo registerGenericUDTF(String functionName, public FunctionInfo registerGenericUDAF(String functionName, GenericUDAFResolver genericUDAFResolver, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerGenericUDAF(functionName, functionType, genericUDAFResolver, resources); + } + + private FunctionInfo registerGenericUDAF(String functionName, FunctionType functionType, + GenericUDAFResolver genericUDAFResolver, FunctionResource... resources) { FunctionInfo function = - new WindowFunctionInfo(isNative, functionName, genericUDAFResolver, resources); + new WindowFunctionInfo(functionType, functionName, genericUDAFResolver, resources); addFunction(functionName, function); addFunction(WINDOW_FUNC_PREFIX + functionName, function); return function; @@ -195,8 +227,14 @@ public FunctionInfo registerGenericUDAF(String functionName, public FunctionInfo registerUDAF(String functionName, Class udafClass, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerUDAF(functionName, functionType, udafClass, resources); + } + + private FunctionInfo registerUDAF(String functionName, FunctionType functionType, + Class udafClass, FunctionResource... resources) { validateClass(udafClass, UDAF.class); - FunctionInfo function = new WindowFunctionInfo(isNative, functionName, + FunctionInfo function = new WindowFunctionInfo(functionType, functionName, new GenericUDAFBridge(ReflectionUtil.newInstance(udafClass, null)), resources); addFunction(functionName, function); addFunction(WINDOW_FUNC_PREFIX + functionName, function); @@ -205,8 +243,14 @@ public FunctionInfo registerUDAF(String functionName, public FunctionInfo registerTableFunction(String functionName, Class tFnCls, FunctionResource... resources) { + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + return registerTableFunction(functionName, functionType, tFnCls, resources); + } + + private FunctionInfo registerTableFunction(String functionName, FunctionType functionType, + Class tFnCls, FunctionResource... resources) { validateClass(tFnCls, TableFunctionResolver.class); - FunctionInfo function = new FunctionInfo(isNative, functionName, tFnCls, resources); + FunctionInfo function = new FunctionInfo(functionType, functionName, tFnCls, resources); addFunction(functionName, function); return function; } @@ -219,7 +263,7 @@ public FunctionInfo registerMacro(String macroName, ExprNodeDesc body, public FunctionInfo registerMacro(String macroName, ExprNodeDesc body, List colNames, List colTypes, FunctionResource... resources) { GenericUDFMacro macro = new GenericUDFMacro(macroName, body, colNames, colTypes); - FunctionInfo fI = new FunctionInfo(isNative, macroName, macro, resources); + FunctionInfo fI = new FunctionInfo(FunctionType.TEMPORARY, macroName, macro, resources); addFunction(macroName, fI); return fI; } @@ -252,7 +296,8 @@ public FunctionInfo registerPermanentFunction(String functionName, * @param wFn */ void registerWindowFunction(String name, GenericUDAFResolver wFn) { - addFunction(WINDOW_FUNC_PREFIX + name, new WindowFunctionInfo(isNative, name, wFn, null)); + FunctionType functionType = isNative ? FunctionType.BUILTIN : FunctionType.TEMPORARY; + addFunction(WINDOW_FUNC_PREFIX + name, new WindowFunctionInfo(functionType, name, wFn, null)); } private void validateClass(Class input, Class expected) { @@ -448,7 +493,10 @@ public GenericUDAFEvaluator getGenericWindowingEvaluator(String functionName, private void addFunction(String functionName, FunctionInfo function) { lock.lock(); try { - if (isNative != function.isNative()) { + // Built-in functions shouldn't go in the session registry, + // and temp functions shouldn't go in the system registry. + // Persistent functions can be in either registry. + if ((!isNative && function.isBuiltIn()) || (isNative && !function.isNative())) { throw new RuntimeException("Function " + functionName + " is not for this registry"); } functionName = functionName.toLowerCase(); @@ -597,7 +645,10 @@ private FunctionInfo registerToSessionRegistry(String qualifiedName, FunctionInf ClassLoader loader = Utilities.getSessionSpecifiedClassLoader(); Class udfClass = Class.forName(function.getClassName(), true, loader); - ret = FunctionRegistry.registerTemporaryUDF(qualifiedName, udfClass, resources); + // Make sure the FunctionInfo is listed as PERSISTENT (rather than TEMPORARY) + // when it is registered to the system registry. + ret = SessionState.getRegistryForWrite().registerFunction( + qualifiedName, FunctionType.PERSISTENT, udfClass, resources); if (ret == null) { LOG.error(function.getClassName() + " is not a valid UDF class and was not registered."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java index a16d9e5..5e57ccd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java @@ -28,9 +28,9 @@ private final boolean pivotResult; private final boolean impliesOrder; - public WindowFunctionInfo(boolean isNative, String functionName, + public WindowFunctionInfo(FunctionType functionType, String functionName, GenericUDAFResolver resolver, FunctionResource[] resources) { - super(isNative, functionName, resolver, resources); + super(functionType, functionName, resolver, resources); WindowFunctionDescription def = AnnotationUtils.getAnnotation(resolver.getClass(), WindowFunctionDescription.class); supportsWindow = def == null ? true : def.supportsWindow(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java index 911b86b..53e4a2a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/SqlFunctionConverter.java @@ -191,7 +191,7 @@ private static FunctionInfo handleCastForParameterizedType(TypeInfo ti, Function throw new RuntimeException(e); } return new FunctionInfo( - fi.isNative(), fi.getDisplayName(), (GenericUDF) udf, fi.getResources()); + fi.getFunctionType(), fi.getDisplayName(), (GenericUDF) udf, fi.getResources()); } // TODO: 1) handle Agg Func Name translation 2) is it correct to add func diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java index d2d5a1b..7fc9ede 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFunctionRegistry.java @@ -27,7 +27,13 @@ import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionResource; +import org.apache.hadoop.hive.ql.exec.FunctionInfo.FunctionType; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.udf.UDFLn; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcat; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @@ -406,4 +412,77 @@ public void testImpliesOrder() throws Exception { Assert.assertTrue(FunctionRegistry.impliesOrder("lag")); Assert.assertFalse(FunctionRegistry.impliesOrder("min")); } + + public void testRegisterTemporaryFunctions() throws Exception { + FunctionResource[] emptyResources = new FunctionResource[] {}; + + // UDF + FunctionRegistry.registerTemporaryUDF("tmp_ln", UDFLn.class, emptyResources); + FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo("tmp_ln"); + assertFalse(functionInfo.isNative()); + + // GenericUDF + FunctionRegistry.registerTemporaryUDF("tmp_concat", GenericUDFConcat.class, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("tmp_concat"); + assertFalse(functionInfo.isNative()); + + // GenericUDAF + FunctionRegistry.registerTemporaryUDF("tmp_max",GenericUDAFMax.class, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("tmp_max"); + assertFalse(functionInfo.isNative()); + functionInfo = FunctionRegistry.getWindowFunctionInfo("tmp_max"); + assertFalse(functionInfo.isNative()); + + // UDTF + FunctionRegistry.registerTemporaryUDF("tmp_explode", GenericUDTFExplode.class, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("tmp_explode"); + assertFalse(functionInfo.isNative()); + } + + public void testRegisterPermanentFunction() throws Exception { + FunctionResource[] emptyResources = new FunctionResource[] {}; + + // UDF + FunctionRegistry.registerPermanentFunction("perm_ln", UDFLn.class.getName(), true, emptyResources); + FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo("perm_ln"); + assertTrue(functionInfo.isPersistent()); + assertTrue(functionInfo.isNative()); + assertFalse(functionInfo.isBuiltIn()); + functionInfo = FunctionRegistry.getFunctionInfo("default.perm_ln"); + assertTrue(functionInfo.isPersistent()); + assertTrue(functionInfo.isNative()); + assertFalse(functionInfo.isBuiltIn()); + + // GenericUDF + FunctionRegistry.registerPermanentFunction("default.perm_concat", + GenericUDFConcat.class.getName(), true, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("default.perm_concat"); + assertTrue(functionInfo.isPersistent()); + assertTrue(functionInfo.isNative()); + assertFalse(functionInfo.isBuiltIn()); + + // GenericUDAF + FunctionRegistry.registerPermanentFunction("default.perm_max", + GenericUDAFMax.class.getName(), true, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("default.perm_max"); + assertTrue(functionInfo.isPersistent()); + functionInfo = FunctionRegistry.getWindowFunctionInfo("default.perm_max"); + assertTrue(functionInfo.isPersistent()); + assertTrue(functionInfo.isNative()); + assertFalse(functionInfo.isBuiltIn()); + + // UDTF + FunctionRegistry.registerPermanentFunction("default.perm_explode", + GenericUDTFExplode.class.getName(), true, emptyResources); + functionInfo = FunctionRegistry.getFunctionInfo("default.perm_explode"); + assertTrue(functionInfo.isPersistent()); + assertTrue(functionInfo.isNative()); + assertFalse(functionInfo.isBuiltIn()); + } + + public void testBuiltInFunction() throws Exception { + FunctionInfo functionInfo = FunctionRegistry.getFunctionInfo("ln"); + assertTrue(functionInfo.isBuiltIn()); + assertTrue(functionInfo.isNative()); + } }