diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 734742c..58aa0f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -189,6 +189,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveRoleGrant; import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveV1Authorizer; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMacro; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.Deserializer; @@ -2780,7 +2781,7 @@ private int describeFunction(DescFunctionDesc descFunc) throws HiveException, SQ desc = AnnotationUtils.getAnnotation(funcClass, Description.class); } if (desc != null) { - outStream.writeBytes(desc.value().replace("_FUNC_", funcName)); + outStream.write(desc.value().replace("_FUNC_", funcName).getBytes("UTF-8")); if (descFunc.isExtended()) { Set synonyms = FunctionRegistry.getFunctionSynonyms(funcName); if (synonyms.size() > 0) { @@ -2792,7 +2793,15 @@ private int describeFunction(DescFunctionDesc descFunc) throws HiveException, SQ } } } else { - if (funcClass != null) { + if (functionInfo.getGenericUDF() instanceof GenericUDFMacro) { + GenericUDFMacro macro = (GenericUDFMacro)functionInfo.getGenericUDF(); + StringBuilder builder = new StringBuilder(); + builder.append(funcName); + builder.append(" (Stateful : ").append(macro.isStateful()); + builder.append(", Deterministic : ").append(macro.isDeterministic()).append(") -\n"); + builder.append(macro.getBodyString().trim()); + outStream.write(builder.toString().getBytes("UTF-8")); + } else if (funcClass != null) { outStream.writeBytes("There is no documentation for function '" + funcName + "'"); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java index b09b706..0127005 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java @@ -166,9 +166,7 @@ public boolean isDeterministic() { public boolean isStateful() { boolean result = FunctionRegistry.isStateful(genericUDF); for (ExprNodeEvaluator child : children) { - if(result = result || child.isStateful()) { - return result; - } + result = result || child.isStateful(); } return result; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index f1fe30d..fabc6d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -1337,8 +1337,8 @@ public static GenericUDF cloneGenericUDF(GenericUDF genericUDF) { bridge.getUdfClassName()); } else if (genericUDF instanceof GenericUDFMacro) { GenericUDFMacro bridge = (GenericUDFMacro) genericUDF; - clonedUDF = new GenericUDFMacro(bridge.getMacroName(), bridge.getBody().clone(), - bridge.getColNames(), bridge.getColTypes()); + clonedUDF = new GenericUDFMacro(bridge.getMacroName(), bridge.getBodyString(), + bridge.getBody().clone(), bridge.getColNames(), bridge.getColTypes()); } else { clonedUDF = ReflectionUtils.newInstance(genericUDF.getClass(), null); } @@ -1549,13 +1549,15 @@ public static void unregisterTemporaryUDF(String functionName) throws HiveExcept * * @param macroName name under which to register the macro * + * @param bodyString the string used to define the macro * @param body the expression which the macro evaluates to - *@param colNames the names of the arguments to the macro - *@param colTypes the types of the arguments to the macro + * @param colNames the names of the arguments to the macro + * @param colTypes the types of the arguments to the macro */ - public static void registerTemporaryMacro( - String macroName, ExprNodeDesc body, List colNames, List colTypes) { - SessionState.getRegistryForWrite().registerMacro(macroName, body, colNames, colTypes); + public static void registerTemporaryMacro(String macroName, String bodyString, + ExprNodeDesc body, List colNames, List colTypes) { + SessionState.getRegistryForWrite().registerMacro( + macroName, bodyString, body, colNames, colTypes); } public static FunctionInfo registerPermanentFunction(String functionName, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java index 7671d29..6e1ca49 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java @@ -186,6 +186,7 @@ private int createTemporaryFunction(CreateFunctionDesc createFunctionDesc) { private int createMacro(CreateMacroDesc createMacroDesc) { FunctionRegistry.registerTemporaryMacro( createMacroDesc.getMacroName(), + createMacroDesc.getBodyString(), createMacroDesc.getBody(), createMacroDesc.getColNames(), createMacroDesc.getColTypes() diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java index a5d59ae..86edae5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -46,7 +46,6 @@ import org.apache.hadoop.util.ReflectionUtils; import java.io.IOException; -import java.net.URLClassLoader; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; @@ -183,14 +182,14 @@ public FunctionInfo registerTableFunction(String functionName, return function; } - public FunctionInfo registerMacro(String macroName, ExprNodeDesc body, + public FunctionInfo registerMacro(String macroName, String bodyString, ExprNodeDesc body, List colNames, List colTypes) { - return registerMacro(macroName, body, colNames, colTypes, null); + return registerMacro(macroName, bodyString, body, colNames, colTypes, new FunctionResource[0]); } - public FunctionInfo registerMacro(String macroName, ExprNodeDesc body, + public FunctionInfo registerMacro(String macroName, String bodyString, ExprNodeDesc body, List colNames, List colTypes, FunctionResource... resources) { - GenericUDFMacro macro = new GenericUDFMacro(macroName, body, colNames, colTypes); + GenericUDFMacro macro = new GenericUDFMacro(macroName, bodyString, body, colNames, colTypes); FunctionInfo fI = new FunctionInfo(isNative, macroName, macro, resources); addFunction(macroName, fI); return fI; @@ -226,7 +225,7 @@ void registerWindowFunction(String name, GenericUDAFResolver wFn) { addFunction(WINDOW_FUNC_PREFIX + name, new WindowFunctionInfo(isNative, name, wFn, null)); } - private void validateClass(Class input, Class expected) { + private void validateClass(Class input, Class expected) { if (!expected.isAssignableFrom(input)) { throw new RuntimeException("Registering UDF Class " + input + " which does not extend " + expected); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java index e3ba201..8a37de2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import org.antlr.runtime.TokenRewriteStream; + import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_IFEXISTS; import java.util.ArrayList; @@ -44,13 +46,10 @@ import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.PreOrderWalker; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.CreateMacroDesc; import org.apache.hadoop.hive.ql.plan.DropMacroDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.FunctionWork; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -132,16 +131,16 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) throw new SemanticException("At least one parameter name was used more than once " + macroColNames); } - SemanticAnalyzer sa = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) ? new CalcitePlanner( - conf) : new SemanticAnalyzer(conf); - ; - ExprNodeDesc body; - if(isNoArgumentMacro) { - body = sa.genExprNodeDesc((ASTNode)ast.getChild(1), rowResolver); - } else { - body = sa.genExprNodeDesc((ASTNode)ast.getChild(2), rowResolver); - } - CreateMacroDesc desc = new CreateMacroDesc(functionName, macroColNames, macroColTypes, body); + SemanticAnalyzer sa = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED) ? + new CalcitePlanner(conf) : new SemanticAnalyzer(conf); + + ASTNode bodyNode = (ASTNode)ast.getChild(isNoArgumentMacro ? 1 : 2); + TokenRewriteStream ts = ctx.getTokenRewriteStream(); + String bodyString = ts.toString(bodyNode.getTokenStartIndex(), Integer.MAX_VALUE); + + ExprNodeDesc body = sa.genExprNodeDesc(bodyNode, rowResolver); + CreateMacroDesc desc = new CreateMacroDesc( + functionName, bodyString, macroColNames, macroColTypes, body); rootTasks.add(TaskFactory.get(new FunctionWork(desc), conf)); addEntities(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java index 443614c..387941f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateMacroDesc.java @@ -33,6 +33,7 @@ private static final long serialVersionUID = 1L; private String macroName; + private String bodyString; private List colNames; private List colTypes; private ExprNodeDesc body; @@ -44,10 +45,12 @@ public CreateMacroDesc() { } public CreateMacroDesc(String macroName, + String bodyString, List colNames, List colTypes, ExprNodeDesc body) { this.macroName = macroName; + this.bodyString = bodyString; this.colNames = colNames; this.colTypes = colTypes; this.body = body; @@ -70,4 +73,11 @@ public ExprNodeDesc getBody() { return colTypes; } + public String getBodyString() { + return bodyString; + } + + public void setBodyString(String bodyString) { + this.bodyString = bodyString; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMacro.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMacro.java index 3f505f2..c42350e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMacro.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMacro.java @@ -42,6 +42,7 @@ private static final long serialVersionUID = 2829755821687181020L; private String macroName; + private String bodyString; private ExprNodeDesc bodyDesc; private transient ExprNodeEvaluator body; private List colNames; @@ -49,10 +50,11 @@ private transient ObjectInspectorConverters.Converter converters[]; private transient ArrayList evaluatedArguments; - public GenericUDFMacro(String macroName, ExprNodeDesc bodyDesc, + public GenericUDFMacro(String macroName, String bodyString, ExprNodeDesc bodyDesc, List colNames, List colTypes) { this.macroName = macroName; + this.bodyString = bodyString; this.bodyDesc = bodyDesc; this.colNames = colNames; this.colTypes = colTypes; @@ -158,4 +160,12 @@ public void setColTypes(List colTypes) { public List getColTypes() { return colTypes; } + + public String getBodyString() { + return bodyString; + } + + public void setBodyString(String bodyString) { + this.bodyString = bodyString; + } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestCreateMacroDesc.java b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestCreateMacroDesc.java index 00a24b9..af90456 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/plan/TestCreateMacroDesc.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/plan/TestCreateMacroDesc.java @@ -45,7 +45,7 @@ public void setup() throws Exception { } @Test public void testCreateMacroDesc() throws Exception { - CreateMacroDesc desc = new CreateMacroDesc(name, colNames, colTypes, bodyDesc); + CreateMacroDesc desc = new CreateMacroDesc(name, "1", colNames, colTypes, bodyDesc); Assert.assertEquals(name, desc.getMacroName()); Assert.assertEquals(bodyDesc, desc.getBody()); Assert.assertEquals(colNames, desc.getColNames()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFMacro.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFMacro.java index 8bbaa6b..76524c5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFMacro.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFMacro.java @@ -75,7 +75,7 @@ public void setup() throws Exception { @Test public void testUDF() throws Exception { - udf = new GenericUDFMacro(name, bodyDesc, colNames, colTypes); + udf = new GenericUDFMacro(name, String.valueOf(expected), bodyDesc, colNames, colTypes); udf.initialize(inspectors); Object actual = udf.evaluate(arguments); Assert.assertEquals(bodyDesc.getValue(), ((IntWritable)actual).get()); diff --git a/ql/src/test/queries/clientpositive/macro.q b/ql/src/test/queries/clientpositive/macro.q index 47b05ff..67c2cd1 100644 --- a/ql/src/test/queries/clientpositive/macro.q +++ b/ql/src/test/queries/clientpositive/macro.q @@ -1,12 +1,16 @@ set hive.fetch.task.conversion=more; CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)); +DESC FUNCTION SIGMOID; + SELECT SIGMOID(2) FROM src LIMIT 1; EXPLAIN SELECT SIGMOID(2) FROM src LIMIT 1; EXPLAIN EXTENDED SELECT SIGMOID(2) FROM src LIMIT 1; DROP TEMPORARY MACRO SIGMOID; CREATE TEMPORARY MACRO FIXED_NUMBER() 1; +DESC FUNCTION FIXED_NUMBER; + SELECT FIXED_NUMBER() + 1 FROM src LIMIT 1; EXPLAIN SELECT FIXED_NUMBER() + 1 FROM src LIMIT 1; EXPLAIN EXTENDED SELECT FIXED_NUMBER() + 1 FROM src LIMIT 1; @@ -14,6 +18,8 @@ DROP TEMPORARY MACRO FIXED_NUMBER; set macrotest=1; CREATE TEMPORARY MACRO CONF_TEST() "${hiveconf:macrotest}"; +DESC FUNCTION CONF_TEST; + SELECT CONF_TEST() FROM src LIMIT 1; DROP TEMPORARY MACRO CONF_TEST; diff --git a/ql/src/test/results/clientpositive/macro.q.out b/ql/src/test/results/clientpositive/macro.q.out index 76ea250..93a36a5 100644 --- a/ql/src/test/results/clientpositive/macro.q.out +++ b/ql/src/test/results/clientpositive/macro.q.out @@ -4,6 +4,12 @@ PREHOOK: Output: database:default POSTHOOK: query: CREATE TEMPORARY MACRO SIGMOID (x DOUBLE) 1.0 / (1.0 + EXP(-x)) POSTHOOK: type: CREATEMACRO POSTHOOK: Output: database:default +PREHOOK: query: DESC FUNCTION SIGMOID +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESC FUNCTION SIGMOID +POSTHOOK: type: DESCFUNCTION +SIGMOID (Stateful : false, Deterministic : true) - +1.0 / (1.0 + EXP(-x)) PREHOOK: query: SELECT SIGMOID(2) FROM src LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -94,6 +100,12 @@ PREHOOK: Output: database:default POSTHOOK: query: CREATE TEMPORARY MACRO FIXED_NUMBER() 1 POSTHOOK: type: CREATEMACRO POSTHOOK: Output: database:default +PREHOOK: query: DESC FUNCTION FIXED_NUMBER +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESC FUNCTION FIXED_NUMBER +POSTHOOK: type: DESCFUNCTION +FIXED_NUMBER (Stateful : false, Deterministic : true) - +1 PREHOOK: query: SELECT FIXED_NUMBER() + 1 FROM src LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -185,6 +197,12 @@ PREHOOK: Output: database:default POSTHOOK: query: CREATE TEMPORARY MACRO CONF_TEST() "1" POSTHOOK: type: CREATEMACRO POSTHOOK: Output: database:default +PREHOOK: query: DESC FUNCTION CONF_TEST +PREHOOK: type: DESCFUNCTION +POSTHOOK: query: DESC FUNCTION CONF_TEST +POSTHOOK: type: DESCFUNCTION +CONF_TEST (Stateful : false, Deterministic : true) - +"1" PREHOOK: query: SELECT CONF_TEST() FROM src LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src