Index: ql/src/test/results/clientpositive/udf_reflect.q.out =================================================================== --- ql/src/test/results/clientpositive/udf_reflect.q.out (revision 0) +++ ql/src/test/results/clientpositive/udf_reflect.q.out (revision 0) @@ -0,0 +1,132 @@ +PREHOOK: query: EXPLAIN EXTENDED +SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1 +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN EXTENDED +SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1 +POSTHOOK: type: QUERY +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.String" "valueOf" 1)) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.String" "isEmpty")) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.Math" "max" 2 3)) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.Math" "min" 2 3)) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.Math" "round" 2.5)) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.Math" "exp" 1.0)) (TOK_SELEXPR (TOK_FUNCTION reflect "java.lang.Math" "floor" 1.9))) (TOK_LIMIT 1))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + src + TableScan + alias: src + Select Operator + expressions: + expr: reflect('java.lang.String','valueOf',1) + type: string + expr: reflect('java.lang.String','isEmpty') + type: string + expr: reflect('java.lang.Math','max',2,3) + type: string + expr: reflect('java.lang.Math','min',2,3) + type: string + expr: reflect('java.lang.Math','round',2.5) + type: string + expr: reflect('java.lang.Math','exp',1.0) + type: string + expr: reflect('java.lang.Math','floor',1.9) + type: string + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Limit + File Output Operator + compressed: false + GlobalTableId: 0 + directory: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-03_23-54-39_660_5582422589206182286/10001 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6 + serialization.format 1 + columns.types string:string:string:string:string:string:string + Needs Tagging: false + Path -> Alias: + file:/data/users/zshao/hadoop_hive_trunk/build/ql/test/data/warehouse/src [src] + Path -> Partition: + file:/data/users/zshao/hadoop_hive_trunk/build/ql/test/data/warehouse/src + Partition + base file name: src + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + name src + columns.types string:string + serialization.ddl struct src { string key, string value} + serialization.format 1 + columns key,value + bucket_count -1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location file:/data/users/zshao/hadoop_hive_trunk/build/ql/test/data/warehouse/src + transient_lastDdlTime 1265270078 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + name src + columns.types string:string + serialization.ddl struct src { string key, string value} + serialization.format 1 + columns key,value + bucket_count -1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + file.inputformat org.apache.hadoop.mapred.TextInputFormat + file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + location file:/data/users/zshao/hadoop_hive_trunk/build/ql/test/data/warehouse/src + transient_lastDdlTime 1265270078 + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: src + name: src + + Stage: Stage-0 + Fetch Operator + limit: 1 + + +PREHOOK: query: SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-03_23-54-39_929_533470423989487198/10000 +POSTHOOK: query: SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-02-03_23-54-39_929_533470423989487198/10000 +1 true 3 2 3 2.7182818284590455 1.0 Index: ql/src/test/queries/clientpositive/udf_reflect.q =================================================================== --- ql/src/test/queries/clientpositive/udf_reflect.q (revision 0) +++ ql/src/test/queries/clientpositive/udf_reflect.q (revision 0) @@ -0,0 +1,19 @@ +EXPLAIN EXTENDED +SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1; + + +SELECT reflect("java.lang.String", "valueOf", 1), + reflect("java.lang.String", "isEmpty"), + reflect("java.lang.Math", "max", 2, 3), + reflect("java.lang.Math", "min", 2, 3), + reflect("java.lang.Math", "round", 2.5), + reflect("java.lang.Math", "exp", 1.0), + reflect("java.lang.Math", "floor", 1.9) +FROM src LIMIT 1; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (revision 906376) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (working copy) @@ -150,6 +150,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMap; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFReflect; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; @@ -320,6 +321,8 @@ registerGenericUDAF("var_samp", new GenericUDAFVarianceSample()); // Generic UDFs + registerGenericUDF("reflect", GenericUDFReflect.class); + registerGenericUDF("array", GenericUDFArray.class); registerGenericUDF("map", GenericUDFMap.class); registerGenericUDF("struct", GenericUDFStruct.class); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java (revision 0) @@ -0,0 +1,178 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.util.ReflectionUtils; + +/** + * A simple generic udf to call java static functions via reflection. + */ +public class GenericUDFReflect extends GenericUDF { + + PrimitiveObjectInspector[] argumentOIs; + StringObjectInspector classNameOI; + StringObjectInspector methodNameOI; + + Class[] parameterJavaClasses; // Classes are Integer, Double, String + Class[] parameterJavaTypes; // Types are int, double, etc + Object[] parameterJavaValues; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + + if (arguments.length < 2) { + throw new UDFArgumentLengthException( + "The function GenericUDFReflect(class,method[,arg1[,arg2]...])" + + " accepts 2 or more arguments."); + } + + for (int i = 0; i < 2; i++) { + if (!(arguments[i] instanceof StringObjectInspector)) { + throw new UDFArgumentTypeException(i, + "The first 2 parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" + + " should be string."); + } + } + + classNameOI = (StringObjectInspector) + ObjectInspectorUtils.getStandardObjectInspector(arguments[0]); + methodNameOI = (StringObjectInspector) + ObjectInspectorUtils.getStandardObjectInspector(arguments[1]); + + parameterJavaClasses = new Class[arguments.length - 2]; + parameterJavaTypes = new Class[arguments.length - 2]; + for (int i = 2; i < arguments.length; i++) { + if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + throw new UDFArgumentTypeException(i, + "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" + + " must be primitive (int, double, string, etc)."); + } + PrimitiveCategory category = + ((PrimitiveObjectInspector)arguments[i]).getPrimitiveCategory(); + PrimitiveTypeEntry t = + PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory(category); + parameterJavaClasses[i - 2] = t.primitiveJavaClass; + parameterJavaTypes[i - 2] = t.primitiveJavaType; + } + + parameterJavaValues = new Object[arguments.length - 2]; + + argumentOIs = new PrimitiveObjectInspector[arguments.length]; + for (int i = 0; i < arguments.length; i++) { + argumentOIs[i] = (PrimitiveObjectInspector)arguments[i]; + } + + return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector( + PrimitiveCategory.STRING); + } + + Class c; + Object o; + Method m; + Object className; + Object methodName; + String result; + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + + // Try to find the class + // Skip class loading if the class name didn't change + boolean classNameChanged = false; + + ObjectInspector newClassNameOI = argumentOIs[0]; + Object newClassName = arguments[0].get(); + + if (className == null || ObjectInspectorUtils.compare(className, classNameOI, newClassName, + newClassNameOI) != 0) { + className = ObjectInspectorUtils.copyToStandardObject(newClassName, newClassNameOI); + String classNameString = classNameOI.getPrimitiveJavaObject(className); + try { + c = Class.forName(classNameString); + } catch (ClassNotFoundException ex) { + throw new HiveException("UDFReflect evaluate ", ex); + } + try { + o = null; + o = ReflectionUtils.newInstance(c, null); + } catch (Exception e) { + // ignored + } + classNameChanged = true; + } + + // Try to find the method + // Skip method finding if the method name didn't change, and class name didn't change. + ObjectInspector newMethodNameOI = argumentOIs[1]; + Object newMethodName = arguments[1].get(); + + if (methodName == null || ObjectInspectorUtils.compare(methodName, methodNameOI, newMethodName, + newMethodNameOI) != 0 || classNameChanged) { + methodName = ObjectInspectorUtils.copyToStandardObject(newMethodName, newMethodNameOI); + String methodNameString = methodNameOI.getPrimitiveJavaObject(methodName); + try { + m = c.getMethod(methodNameString, parameterJavaClasses); + } catch (SecurityException e) { + throw new HiveException("UDFReflect getMethod ", e); + } catch (NoSuchMethodException e) { + try { + m = c.getMethod(methodNameString, parameterJavaTypes); + } catch (SecurityException ex) { + throw new HiveException("UDFReflect getMethod ", ex); + } catch (NoSuchMethodException ex) { + throw new HiveException("UDFReflect getMethod ", ex); + } + } + } + + // Get the parameter values + for (int i = 2; i < arguments.length; i++) { + parameterJavaValues[i - 2] = argumentOIs[i].getPrimitiveJavaObject(arguments[i].get()); + } + + try { + result = String.valueOf(m.invoke(o, parameterJavaValues)); + return result; + } catch (IllegalArgumentException e1) { + System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + + Arrays.asList(parameterJavaValues)); + } catch (IllegalAccessException e1) { + System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + + Arrays.asList(parameterJavaValues)); + } catch (InvocationTargetException e1) { + System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " + + Arrays.asList(parameterJavaValues)); + } + return null; + } + + @Override + public String getDisplayString(String[] children) { + StringBuilder sb = new StringBuilder(); + sb.append("reflect("); + for (int i = 0; i < children.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(children[i]); + } + sb.append(')'); + return sb.toString(); + } + +}