diff --git contrib/src/test/results/clientnegative/invalid_row_sequence.q.out contrib/src/test/results/clientnegative/invalid_row_sequence.q.out index 8f3c0b3..15b96c2 100644 --- contrib/src/test/results/clientnegative/invalid_row_sequence.q.out +++ contrib/src/test/results/clientnegative/invalid_row_sequence.q.out @@ -2,12 +2,10 @@ PREHOOK: query: -- Verify that a stateful UDF cannot be used outside of the SELE drop temporary function row_sequence PREHOOK: type: DROPFUNCTION -PREHOOK: Output: row_sequence POSTHOOK: query: -- Verify that a stateful UDF cannot be used outside of the SELECT list drop temporary function row_sequence POSTHOOK: type: DROPFUNCTION -POSTHOOK: Output: row_sequence PREHOOK: query: create temporary function row_sequence as 'org.apache.hadoop.hive.contrib.udf.UDFRowSequence' PREHOOK: type: CREATEFUNCTION diff --git contrib/src/test/results/clientpositive/udaf_example_max.q.out contrib/src/test/results/clientpositive/udaf_example_max.q.out index 82aeca7..5f8855b 100644 --- contrib/src/test/results/clientpositive/udaf_example_max.q.out +++ contrib/src/test/results/clientpositive/udaf_example_max.q.out @@ -9,6 +9,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED example_max POSTHOOK: type: DESCFUNCTION example_max(expr) - Returns the maximum value of expr +Synonyms: @_example_max PREHOOK: query: EXPLAIN SELECT example_max(substr(value,5)), example_max(IF(substr(value,5) > 250, NULL, substr(value,5))) diff --git contrib/src/test/results/clientpositive/udaf_example_min.q.out contrib/src/test/results/clientpositive/udaf_example_min.q.out index b62ff39..5dd2b33 100644 --- contrib/src/test/results/clientpositive/udaf_example_min.q.out +++ contrib/src/test/results/clientpositive/udaf_example_min.q.out @@ -9,6 +9,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED example_min POSTHOOK: type: DESCFUNCTION example_min(expr) - Returns the minimum value of expr +Synonyms: @_example_min PREHOOK: query: EXPLAIN SELECT example_min(substr(value,5)), example_min(IF(substr(value,5) > 250, NULL, substr(value,5))) diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index f5e35b8..479abe0 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -630,8 +630,8 @@ public void cleanUp() throws Exception { // Best effort } - FunctionRegistry.unregisterTemporaryUDF("test_udaf"); - FunctionRegistry.unregisterTemporaryUDF("test_error"); + FunctionRegistry.unregisterTemporaryFunction("test_udaf"); + FunctionRegistry.unregisterTemporaryFunction("test_error"); } private void runLoadCmd(String loadCmd) throws Exception { diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 31978fe..f0327e0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -421,6 +421,7 @@ "an AcidOutputFormat or is not bucketed", true), ACID_NO_SORTED_BUCKETS(10298, "ACID insert, update, delete not supported on tables that are " + "sorted, table {0}", true), + DROP_NATIVE_FUNCTION(10299, "Cannot drop native function"), //========================== 20000 range starts here ========================// SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."), diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java deleted file mode 100644 index 93c15c0..0000000 --- ql/src/java/org/apache/hadoop/hive/ql/exec/CommonFunctionInfo.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec; - -/** - * Interface for common functionality between FunctionInfo/WindowFunctionInfo - */ -public interface CommonFunctionInfo { - Class getFunctionClass(); -} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java index 074255b..73646a4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionInfo.java @@ -31,7 +31,7 @@ * FunctionInfo. * */ -public class FunctionInfo implements CommonFunctionInfo { +public class FunctionInfo { private final boolean isNative; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index a9c85f8..5aee13e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -18,34 +18,19 @@ package org.apache.hadoop.hive.ql.exec; -import java.io.InputStream; import java.lang.reflect.Method; -import java.net.URL; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hive.common.type.HiveDecimal; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.ql.exec.FunctionUtils.UDFClassType; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -141,27 +126,16 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.common.util.AnnotationUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - /** * FunctionRegistry. */ public final class FunctionRegistry { - private static Log LOG = LogFactory.getLog(FunctionRegistry.class); - - /** - * The mapping from expression function names to expression classes. - */ - static Map mFunctions = Collections.synchronizedMap(new LinkedHashMap()); + private static final Log LOG = LogFactory.getLog(FunctionRegistry.class); - static Set> nativeUdfs = Collections.synchronizedSet(new HashSet>()); /* * PTF variables * */ @@ -176,557 +150,322 @@ private static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap"; private static final String NOOP_STREAMING_TABLE_FUNCTION = "noopstreaming"; private static final String NOOP_STREAMING_MAP_TABLE_FUNCTION = "noopwithmapstreaming"; + private static final String MATCH_PATH_TABLE_FUNCTION = "matchpath"; - static Map windowFunctions = Collections.synchronizedMap(new LinkedHashMap()); + // registry for system functions + private static final Registry system = new Registry(true); static { - registerGenericUDF("concat", GenericUDFConcat.class); - registerUDF("substr", UDFSubstr.class, false); - registerUDF("substring", UDFSubstr.class, false); - registerUDF("space", UDFSpace.class, false); - registerUDF("repeat", UDFRepeat.class, false); - registerUDF("ascii", UDFAscii.class, false); - registerGenericUDF("lpad", GenericUDFLpad.class); - registerGenericUDF("rpad", GenericUDFRpad.class); - - registerGenericUDF("size", GenericUDFSize.class); - - registerGenericUDF("round", GenericUDFRound.class); - registerGenericUDF("floor", GenericUDFFloor.class); - registerUDF("sqrt", UDFSqrt.class, false); - registerGenericUDF("ceil", GenericUDFCeil.class); - registerGenericUDF("ceiling", GenericUDFCeil.class); - registerUDF("rand", UDFRand.class, false); - registerGenericUDF("abs", GenericUDFAbs.class); - registerGenericUDF("pmod", GenericUDFPosMod.class); - - registerUDF("ln", UDFLn.class, false); - registerUDF("log2", UDFLog2.class, false); - registerUDF("sin", UDFSin.class, false); - registerUDF("asin", UDFAsin.class, false); - registerUDF("cos", UDFCos.class, false); - registerUDF("acos", UDFAcos.class, false); - registerUDF("log10", UDFLog10.class, false); - registerUDF("log", UDFLog.class, false); - registerUDF("exp", UDFExp.class, false); - registerGenericUDF("power", GenericUDFPower.class); - registerGenericUDF("pow", GenericUDFPower.class); - registerUDF("sign", UDFSign.class, false); - registerUDF("pi", UDFPI.class, false); - registerUDF("degrees", UDFDegrees.class, false); - registerUDF("radians", UDFRadians.class, false); - registerUDF("atan", UDFAtan.class, false); - registerUDF("tan", UDFTan.class, false); - registerUDF("e", UDFE.class, false); - - registerUDF("conv", UDFConv.class, false); - registerUDF("bin", UDFBin.class, false); - registerUDF("hex", UDFHex.class, false); - registerUDF("unhex", UDFUnhex.class, false); - registerUDF("base64", UDFBase64.class, false); - registerUDF("unbase64", UDFUnbase64.class, false); - - registerGenericUDF("encode", GenericUDFEncode.class); - registerGenericUDF("decode", GenericUDFDecode.class); - - registerGenericUDF("upper", GenericUDFUpper.class); - registerGenericUDF("lower", GenericUDFLower.class); - registerGenericUDF("ucase", GenericUDFUpper.class); - registerGenericUDF("lcase", GenericUDFLower.class); - registerGenericUDF("trim", GenericUDFTrim.class); - registerGenericUDF("ltrim", GenericUDFLTrim.class); - registerGenericUDF("rtrim", GenericUDFRTrim.class); - registerUDF("length", UDFLength.class, false); - registerUDF("reverse", UDFReverse.class, false); - registerGenericUDF("field", GenericUDFField.class); - registerUDF("find_in_set", UDFFindInSet.class, false); - - registerUDF("like", UDFLike.class, true); - registerUDF("rlike", UDFRegExp.class, true); - registerUDF("regexp", UDFRegExp.class, true); - registerUDF("regexp_replace", UDFRegExpReplace.class, false); - registerUDF("regexp_extract", UDFRegExpExtract.class, false); - registerUDF("parse_url", UDFParseUrl.class, false); - registerGenericUDF("nvl", GenericUDFNvl.class); - registerGenericUDF("split", GenericUDFSplit.class); - registerGenericUDF("str_to_map", GenericUDFStringToMap.class); - registerGenericUDF("translate", GenericUDFTranslate.class); - - registerGenericUDF("positive", GenericUDFOPPositive.class); - registerGenericUDF("negative", GenericUDFOPNegative.class); - - registerUDF("day", UDFDayOfMonth.class, false); - registerUDF("dayofmonth", UDFDayOfMonth.class, false); - registerUDF("month", UDFMonth.class, false); - registerUDF("year", UDFYear.class, false); - registerUDF("hour", UDFHour.class, false); - registerUDF("minute", UDFMinute.class, false); - registerUDF("second", UDFSecond.class, false); - registerUDF("from_unixtime", UDFFromUnixTime.class, false); - registerGenericUDF("to_date", GenericUDFDate.class); - registerUDF("weekofyear", UDFWeekOfYear.class, false); - - registerGenericUDF("date_add", GenericUDFDateAdd.class); - registerGenericUDF("date_sub", GenericUDFDateSub.class); - registerGenericUDF("datediff", GenericUDFDateDiff.class); - - registerUDF("get_json_object", UDFJson.class, false); - - registerUDF("xpath_string", UDFXPathString.class, false); - registerUDF("xpath_boolean", UDFXPathBoolean.class, false); - registerUDF("xpath_number", UDFXPathDouble.class, false); - registerUDF("xpath_double", UDFXPathDouble.class, false); - registerUDF("xpath_float", UDFXPathFloat.class, false); - registerUDF("xpath_long", UDFXPathLong.class, false); - registerUDF("xpath_int", UDFXPathInteger.class, false); - registerUDF("xpath_short", UDFXPathShort.class, false); - registerGenericUDF("xpath", GenericUDFXPath.class); - - registerGenericUDF("+", GenericUDFOPPlus.class); - registerGenericUDF("-", GenericUDFOPMinus.class); - registerGenericUDF("*", GenericUDFOPMultiply.class); - registerGenericUDF("/", GenericUDFOPDivide.class); - registerGenericUDF("%", GenericUDFOPMod.class); - registerUDF("div", UDFOPLongDivide.class, true); - - registerUDF("&", UDFOPBitAnd.class, true); - registerUDF("|", UDFOPBitOr.class, true); - registerUDF("^", UDFOPBitXor.class, true); - registerUDF("~", UDFOPBitNot.class, true); - - registerGenericUDF("current_database", UDFCurrentDB.class); - - registerGenericUDF("isnull", GenericUDFOPNull.class); - registerGenericUDF("isnotnull", GenericUDFOPNotNull.class); - - registerGenericUDF("if", GenericUDFIf.class); - registerGenericUDF("in", GenericUDFIn.class); - registerGenericUDF("and", GenericUDFOPAnd.class); - registerGenericUDF("or", GenericUDFOPOr.class); - registerGenericUDF("=", GenericUDFOPEqual.class); - registerGenericUDF("==", GenericUDFOPEqual.class); - registerGenericUDF("<=>", GenericUDFOPEqualNS.class); - registerGenericUDF("!=", GenericUDFOPNotEqual.class); - registerGenericUDF("<>", GenericUDFOPNotEqual.class); - registerGenericUDF("<", GenericUDFOPLessThan.class); - registerGenericUDF("<=", GenericUDFOPEqualOrLessThan.class); - registerGenericUDF(">", GenericUDFOPGreaterThan.class); - registerGenericUDF(">=", GenericUDFOPEqualOrGreaterThan.class); - registerGenericUDF("not", GenericUDFOPNot.class); - registerGenericUDF("!", GenericUDFOPNot.class); - registerGenericUDF("between", GenericUDFBetween.class); - - registerGenericUDF("ewah_bitmap_and", GenericUDFEWAHBitmapAnd.class); - registerGenericUDF("ewah_bitmap_or", GenericUDFEWAHBitmapOr.class); - registerGenericUDF("ewah_bitmap_empty", GenericUDFEWAHBitmapEmpty.class); + system.registerGenericUDF("concat", GenericUDFConcat.class, true); + system.registerUDF("substr", UDFSubstr.class, false, true); + system.registerUDF("substring", UDFSubstr.class, false, true); + system.registerUDF("space", UDFSpace.class, false, true); + system.registerUDF("repeat", UDFRepeat.class, false, true); + system.registerUDF("ascii", UDFAscii.class, false, true); + system.registerGenericUDF("lpad", GenericUDFLpad.class, true); + system.registerGenericUDF("rpad", GenericUDFRpad.class, true); + + system.registerGenericUDF("size", GenericUDFSize.class, true); + + system.registerGenericUDF("round", GenericUDFRound.class, true); + system.registerGenericUDF("floor", GenericUDFFloor.class, true); + system.registerUDF("sqrt", UDFSqrt.class, false, true); + system.registerGenericUDF("ceil", GenericUDFCeil.class, true); + system.registerGenericUDF("ceiling", GenericUDFCeil.class, true); + system.registerUDF("rand", UDFRand.class, false, true); + system.registerGenericUDF("abs", GenericUDFAbs.class, true); + system.registerGenericUDF("pmod", GenericUDFPosMod.class, true); + + system.registerUDF("ln", UDFLn.class, false, true); + system.registerUDF("log2", UDFLog2.class, false, true); + system.registerUDF("sin", UDFSin.class, false, true); + system.registerUDF("asin", UDFAsin.class, false, true); + system.registerUDF("cos", UDFCos.class, false, true); + system.registerUDF("acos", UDFAcos.class, false, true); + system.registerUDF("log10", UDFLog10.class, false, true); + system.registerUDF("log", UDFLog.class, false, true); + system.registerUDF("exp", UDFExp.class, false, true); + system.registerGenericUDF("power", GenericUDFPower.class, true); + system.registerGenericUDF("pow", GenericUDFPower.class, true); + system.registerUDF("sign", UDFSign.class, false, true); + system.registerUDF("pi", UDFPI.class, false, true); + system.registerUDF("degrees", UDFDegrees.class, false, true); + system.registerUDF("radians", UDFRadians.class, false, true); + system.registerUDF("atan", UDFAtan.class, false, true); + system.registerUDF("tan", UDFTan.class, false, true); + system.registerUDF("e", UDFE.class, false, true); + + system.registerUDF("conv", UDFConv.class, false, true); + system.registerUDF("bin", UDFBin.class, false, true); + system.registerUDF("hex", UDFHex.class, false, true); + system.registerUDF("unhex", UDFUnhex.class, false, true); + system.registerUDF("base64", UDFBase64.class, false, true); + system.registerUDF("unbase64", UDFUnbase64.class, false, true); + + system.registerGenericUDF("encode", GenericUDFEncode.class, true); + system.registerGenericUDF("decode", GenericUDFDecode.class, true); + + system.registerGenericUDF("upper", GenericUDFUpper.class, true); + system.registerGenericUDF("lower", GenericUDFLower.class, true); + system.registerGenericUDF("ucase", GenericUDFUpper.class, true); + system.registerGenericUDF("lcase", GenericUDFLower.class, true); + system.registerGenericUDF("trim", GenericUDFTrim.class, true); + system.registerGenericUDF("ltrim", GenericUDFLTrim.class, true); + system.registerGenericUDF("rtrim", GenericUDFRTrim.class, true); + system.registerUDF("length", UDFLength.class, false, true); + system.registerUDF("reverse", UDFReverse.class, false, true); + system.registerGenericUDF("field", GenericUDFField.class, true); + system.registerUDF("find_in_set", UDFFindInSet.class, false, true); + + system.registerUDF("like", UDFLike.class, true, true); + system.registerUDF("rlike", UDFRegExp.class, true, true); + system.registerUDF("regexp", UDFRegExp.class, true, true); + system.registerUDF("regexp_replace", UDFRegExpReplace.class, false, true); + system.registerUDF("regexp_extract", UDFRegExpExtract.class, false, true); + system.registerUDF("parse_url", UDFParseUrl.class, false, true); + system.registerGenericUDF("nvl", GenericUDFNvl.class, true); + system.registerGenericUDF("split", GenericUDFSplit.class, true); + system.registerGenericUDF("str_to_map", GenericUDFStringToMap.class, true); + system.registerGenericUDF("translate", GenericUDFTranslate.class, true); + + system.registerGenericUDF("positive", GenericUDFOPPositive.class, true); + system.registerGenericUDF("negative", GenericUDFOPNegative.class, true); + + system.registerUDF("day", UDFDayOfMonth.class, false, true); + system.registerUDF("dayofmonth", UDFDayOfMonth.class, false, true); + system.registerUDF("month", UDFMonth.class, false, true); + system.registerUDF("year", UDFYear.class, false, true); + system.registerUDF("hour", UDFHour.class, false, true); + system.registerUDF("minute", UDFMinute.class, false, true); + system.registerUDF("second", UDFSecond.class, false, true); + system.registerUDF("from_unixtime", UDFFromUnixTime.class, false, true); + system.registerGenericUDF("to_date", GenericUDFDate.class, true); + system.registerUDF("weekofyear", UDFWeekOfYear.class, false, true); + + system.registerGenericUDF("date_add", GenericUDFDateAdd.class, true); + system.registerGenericUDF("date_sub", GenericUDFDateSub.class, true); + system.registerGenericUDF("datediff", GenericUDFDateDiff.class, true); + + system.registerUDF("get_json_object", UDFJson.class, false, true); + + system.registerUDF("xpath_string", UDFXPathString.class, false, true); + system.registerUDF("xpath_boolean", UDFXPathBoolean.class, false, true); + system.registerUDF("xpath_number", UDFXPathDouble.class, false, true); + system.registerUDF("xpath_double", UDFXPathDouble.class, false, true); + system.registerUDF("xpath_float", UDFXPathFloat.class, false, true); + system.registerUDF("xpath_long", UDFXPathLong.class, false, true); + system.registerUDF("xpath_int", UDFXPathInteger.class, false, true); + system.registerUDF("xpath_short", UDFXPathShort.class, false, true); + system.registerGenericUDF("xpath", GenericUDFXPath.class, true); + + system.registerGenericUDF("+", GenericUDFOPPlus.class, true); + system.registerGenericUDF("-", GenericUDFOPMinus.class, true); + system.registerGenericUDF("*", GenericUDFOPMultiply.class, true); + system.registerGenericUDF("/", GenericUDFOPDivide.class, true); + system.registerGenericUDF("%", GenericUDFOPMod.class, true); + system.registerUDF("div", UDFOPLongDivide.class, true, true); + + system.registerUDF("&", UDFOPBitAnd.class, true, true); + system.registerUDF("|", UDFOPBitOr.class, true, true); + system.registerUDF("^", UDFOPBitXor.class, true, true); + system.registerUDF("~", UDFOPBitNot.class, true, true); + + system.registerGenericUDF("current_database", UDFCurrentDB.class, true); + + system.registerGenericUDF("isnull", GenericUDFOPNull.class, true); + system.registerGenericUDF("isnotnull", GenericUDFOPNotNull.class, true); + + system.registerGenericUDF("if", GenericUDFIf.class, true); + system.registerGenericUDF("in", GenericUDFIn.class, true); + system.registerGenericUDF("and", GenericUDFOPAnd.class, true); + system.registerGenericUDF("or", GenericUDFOPOr.class, true); + system.registerGenericUDF("=", GenericUDFOPEqual.class, true); + system.registerGenericUDF("==", GenericUDFOPEqual.class, true); + system.registerGenericUDF("<=>", GenericUDFOPEqualNS.class, true); + system.registerGenericUDF("!=", GenericUDFOPNotEqual.class, true); + system.registerGenericUDF("<>", GenericUDFOPNotEqual.class, true); + system.registerGenericUDF("<", GenericUDFOPLessThan.class, true); + system.registerGenericUDF("<=", GenericUDFOPEqualOrLessThan.class, true); + system.registerGenericUDF(">", GenericUDFOPGreaterThan.class, true); + system.registerGenericUDF(">=", GenericUDFOPEqualOrGreaterThan.class, true); + system.registerGenericUDF("not", GenericUDFOPNot.class, true); + system.registerGenericUDF("!", GenericUDFOPNot.class, true); + system.registerGenericUDF("between", GenericUDFBetween.class, true); + + system.registerGenericUDF("ewah_bitmap_and", GenericUDFEWAHBitmapAnd.class, true); + system.registerGenericUDF("ewah_bitmap_or", GenericUDFEWAHBitmapOr.class, true); + system.registerGenericUDF("ewah_bitmap_empty", GenericUDFEWAHBitmapEmpty.class, true); // Aliases for Java Class Names // These are used in getImplicitConvertUDFMethod - registerUDF(serdeConstants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, false, - UDFToBoolean.class.getSimpleName()); - registerUDF(serdeConstants.TINYINT_TYPE_NAME, UDFToByte.class, false, - UDFToByte.class.getSimpleName()); - registerUDF(serdeConstants.SMALLINT_TYPE_NAME, UDFToShort.class, false, - UDFToShort.class.getSimpleName()); - registerUDF(serdeConstants.INT_TYPE_NAME, UDFToInteger.class, false, - UDFToInteger.class.getSimpleName()); - registerUDF(serdeConstants.BIGINT_TYPE_NAME, UDFToLong.class, false, - UDFToLong.class.getSimpleName()); - registerUDF(serdeConstants.FLOAT_TYPE_NAME, UDFToFloat.class, false, - UDFToFloat.class.getSimpleName()); - registerUDF(serdeConstants.DOUBLE_TYPE_NAME, UDFToDouble.class, false, - UDFToDouble.class.getSimpleName()); - registerUDF(serdeConstants.STRING_TYPE_NAME, UDFToString.class, false, - UDFToString.class.getSimpleName()); - - registerGenericUDF(serdeConstants.DATE_TYPE_NAME, - GenericUDFToDate.class); - registerGenericUDF(serdeConstants.TIMESTAMP_TYPE_NAME, - GenericUDFTimestamp.class); - registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, - GenericUDFToBinary.class); - registerGenericUDF(serdeConstants.DECIMAL_TYPE_NAME, - GenericUDFToDecimal.class); - registerGenericUDF(serdeConstants.VARCHAR_TYPE_NAME, - GenericUDFToVarchar.class); - registerGenericUDF(serdeConstants.CHAR_TYPE_NAME, - GenericUDFToChar.class); + system.registerUDF(serdeConstants.BOOLEAN_TYPE_NAME, UDFToBoolean.class, false, + UDFToBoolean.class.getSimpleName(), true); + system.registerUDF(serdeConstants.TINYINT_TYPE_NAME, UDFToByte.class, false, + UDFToByte.class.getSimpleName(), true); + system.registerUDF(serdeConstants.SMALLINT_TYPE_NAME, UDFToShort.class, false, + UDFToShort.class.getSimpleName(), true); + system.registerUDF(serdeConstants.INT_TYPE_NAME, UDFToInteger.class, false, + UDFToInteger.class.getSimpleName(), true); + system.registerUDF(serdeConstants.BIGINT_TYPE_NAME, UDFToLong.class, false, + UDFToLong.class.getSimpleName(), true); + system.registerUDF(serdeConstants.FLOAT_TYPE_NAME, UDFToFloat.class, false, + UDFToFloat.class.getSimpleName(), true); + system.registerUDF(serdeConstants.DOUBLE_TYPE_NAME, UDFToDouble.class, false, + UDFToDouble.class.getSimpleName(), true); + system.registerUDF(serdeConstants.STRING_TYPE_NAME, UDFToString.class, false, + UDFToString.class.getSimpleName(), true); + + system.registerGenericUDF(serdeConstants.DATE_TYPE_NAME, + GenericUDFToDate.class, true); + system.registerGenericUDF(serdeConstants.TIMESTAMP_TYPE_NAME, + GenericUDFTimestamp.class, true); + system.registerGenericUDF(serdeConstants.BINARY_TYPE_NAME, + GenericUDFToBinary.class, true); + system.registerGenericUDF(serdeConstants.DECIMAL_TYPE_NAME, + GenericUDFToDecimal.class, true); + system.registerGenericUDF(serdeConstants.VARCHAR_TYPE_NAME, + GenericUDFToVarchar.class, true); + system.registerGenericUDF(serdeConstants.CHAR_TYPE_NAME, + GenericUDFToChar.class, true); // Aggregate functions - registerGenericUDAF("max", new GenericUDAFMax()); - registerGenericUDAF("min", new GenericUDAFMin()); + system.registerGenericUDAF("max", new GenericUDAFMax(), true); + system.registerGenericUDAF("min", new GenericUDAFMin(), true); - registerGenericUDAF("sum", new GenericUDAFSum()); - registerGenericUDAF("count", new GenericUDAFCount()); - registerGenericUDAF("avg", new GenericUDAFAverage()); - registerGenericUDAF("std", new GenericUDAFStd()); - registerGenericUDAF("stddev", new GenericUDAFStd()); - registerGenericUDAF("stddev_pop", new GenericUDAFStd()); - registerGenericUDAF("stddev_samp", new GenericUDAFStdSample()); - registerGenericUDAF("variance", new GenericUDAFVariance()); - registerGenericUDAF("var_pop", new GenericUDAFVariance()); - registerGenericUDAF("var_samp", new GenericUDAFVarianceSample()); - registerGenericUDAF("covar_pop", new GenericUDAFCovariance()); - registerGenericUDAF("covar_samp", new GenericUDAFCovarianceSample()); - registerGenericUDAF("corr", new GenericUDAFCorrelation()); - registerGenericUDAF("histogram_numeric", new GenericUDAFHistogramNumeric()); - registerGenericUDAF("percentile_approx", new GenericUDAFPercentileApprox()); - registerGenericUDAF("collect_set", new GenericUDAFCollectSet()); - registerGenericUDAF("collect_list", new GenericUDAFCollectList()); + system.registerGenericUDAF("sum", new GenericUDAFSum(), true); + system.registerGenericUDAF("count", new GenericUDAFCount(), true); + system.registerGenericUDAF("avg", new GenericUDAFAverage(), true); + system.registerGenericUDAF("std", new GenericUDAFStd(), true); + system.registerGenericUDAF("stddev", new GenericUDAFStd(), true); + system.registerGenericUDAF("stddev_pop", new GenericUDAFStd(), true); + system.registerGenericUDAF("stddev_samp", new GenericUDAFStdSample(), true); + system.registerGenericUDAF("variance", new GenericUDAFVariance(), true); + system.registerGenericUDAF("var_pop", new GenericUDAFVariance(), true); + system.registerGenericUDAF("var_samp", new GenericUDAFVarianceSample(), true); + system.registerGenericUDAF("covar_pop", new GenericUDAFCovariance(), true); + system.registerGenericUDAF("covar_samp", new GenericUDAFCovarianceSample(), true); + system.registerGenericUDAF("corr", new GenericUDAFCorrelation(), true); + system.registerGenericUDAF("histogram_numeric", new GenericUDAFHistogramNumeric(), true); + system.registerGenericUDAF("percentile_approx", new GenericUDAFPercentileApprox(), true); + system.registerGenericUDAF("collect_set", new GenericUDAFCollectSet(), true); + system.registerGenericUDAF("collect_list", new GenericUDAFCollectList(), true); - registerGenericUDAF("ngrams", new GenericUDAFnGrams()); - registerGenericUDAF("context_ngrams", new GenericUDAFContextNGrams()); + system.registerGenericUDAF("ngrams", new GenericUDAFnGrams(), true); + system.registerGenericUDAF("context_ngrams", new GenericUDAFContextNGrams(), true); - registerGenericUDAF("ewah_bitmap", new GenericUDAFEWAHBitmap()); + system.registerGenericUDAF("ewah_bitmap", new GenericUDAFEWAHBitmap(), true); - registerGenericUDAF("compute_stats" , new GenericUDAFComputeStats()); + system.registerGenericUDAF("compute_stats" , new GenericUDAFComputeStats(), true); - registerUDAF("percentile", UDAFPercentile.class); + system.registerUDAF("percentile", UDAFPercentile.class, true); // Generic UDFs - registerGenericUDF("reflect", GenericUDFReflect.class); - registerGenericUDF("reflect2", GenericUDFReflect2.class); - registerGenericUDF("java_method", GenericUDFReflect.class); - - registerGenericUDF("array", GenericUDFArray.class); - registerGenericUDF("assert_true", GenericUDFAssertTrue.class); - registerGenericUDF("map", GenericUDFMap.class); - registerGenericUDF("struct", GenericUDFStruct.class); - registerGenericUDF("named_struct", GenericUDFNamedStruct.class); - registerGenericUDF("create_union", GenericUDFUnion.class); - - registerGenericUDF("case", GenericUDFCase.class); - registerGenericUDF("when", GenericUDFWhen.class); - registerGenericUDF("hash", GenericUDFHash.class); - registerGenericUDF("coalesce", GenericUDFCoalesce.class); - registerGenericUDF("index", GenericUDFIndex.class); - registerGenericUDF("in_file", GenericUDFInFile.class); - registerGenericUDF("instr", GenericUDFInstr.class); - registerGenericUDF("locate", GenericUDFLocate.class); - registerGenericUDF("elt", GenericUDFElt.class); - registerGenericUDF("concat_ws", GenericUDFConcatWS.class); - registerGenericUDF("sort_array", GenericUDFSortArray.class); - registerGenericUDF("array_contains", GenericUDFArrayContains.class); - registerGenericUDF("sentences", GenericUDFSentences.class); - registerGenericUDF("map_keys", GenericUDFMapKeys.class); - registerGenericUDF("map_values", GenericUDFMapValues.class); - registerGenericUDF("format_number", GenericUDFFormatNumber.class); - registerGenericUDF("printf", GenericUDFPrintf.class); - - registerGenericUDF("from_utc_timestamp", GenericUDFFromUtcTimestamp.class); - registerGenericUDF("to_utc_timestamp", GenericUDFToUtcTimestamp.class); - - registerGenericUDF("unix_timestamp", GenericUDFUnixTimeStamp.class); - registerGenericUDF("to_unix_timestamp", GenericUDFToUnixTimeStamp.class); + system.registerGenericUDF("reflect", GenericUDFReflect.class, true); + system.registerGenericUDF("reflect2", GenericUDFReflect2.class, true); + system.registerGenericUDF("java_method", GenericUDFReflect.class, true); + + system.registerGenericUDF("array", GenericUDFArray.class, true); + system.registerGenericUDF("assert_true", GenericUDFAssertTrue.class, true); + system.registerGenericUDF("map", GenericUDFMap.class, true); + system.registerGenericUDF("struct", GenericUDFStruct.class, true); + system.registerGenericUDF("named_struct", GenericUDFNamedStruct.class, true); + system.registerGenericUDF("create_union", GenericUDFUnion.class, true); + + system.registerGenericUDF("case", GenericUDFCase.class, true); + system.registerGenericUDF("when", GenericUDFWhen.class, true); + system.registerGenericUDF("hash", GenericUDFHash.class, true); + system.registerGenericUDF("coalesce", GenericUDFCoalesce.class, true); + system.registerGenericUDF("index", GenericUDFIndex.class, true); + system.registerGenericUDF("in_file", GenericUDFInFile.class, true); + system.registerGenericUDF("instr", GenericUDFInstr.class, true); + system.registerGenericUDF("locate", GenericUDFLocate.class, true); + system.registerGenericUDF("elt", GenericUDFElt.class, true); + system.registerGenericUDF("concat_ws", GenericUDFConcatWS.class, true); + system.registerGenericUDF("sort_array", GenericUDFSortArray.class, true); + system.registerGenericUDF("array_contains", GenericUDFArrayContains.class, true); + system.registerGenericUDF("sentences", GenericUDFSentences.class, true); + system.registerGenericUDF("map_keys", GenericUDFMapKeys.class, true); + system.registerGenericUDF("map_values", GenericUDFMapValues.class, true); + system.registerGenericUDF("format_number", GenericUDFFormatNumber.class, true); + system.registerGenericUDF("printf", GenericUDFPrintf.class, true); + + system.registerGenericUDF("from_utc_timestamp", GenericUDFFromUtcTimestamp.class, true); + system.registerGenericUDF("to_utc_timestamp", GenericUDFToUtcTimestamp.class, true); + + system.registerGenericUDF("unix_timestamp", GenericUDFUnixTimeStamp.class, true); + system.registerGenericUDF("to_unix_timestamp", GenericUDFToUnixTimeStamp.class, true); // Generic UDTF's - registerGenericUDTF("explode", GenericUDTFExplode.class); - registerGenericUDTF("inline", GenericUDTFInline.class); - registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class); - registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class); - registerGenericUDTF("posexplode", GenericUDTFPosExplode.class); - registerGenericUDTF("stack", GenericUDTFStack.class); + system.registerGenericUDTF("explode", GenericUDTFExplode.class, true); + system.registerGenericUDTF("inline", GenericUDTFInline.class, true); + system.registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class, true); + system.registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class, true); + system.registerGenericUDTF("posexplode", GenericUDTFPosExplode.class, true); + system.registerGenericUDTF("stack", GenericUDTFStack.class, true); //PTF declarations - registerGenericUDF(LEAD_FUNC_NAME, GenericUDFLead.class); - registerGenericUDF(LAG_FUNC_NAME, GenericUDFLag.class); - - registerWindowFunction("row_number", new GenericUDAFRowNumber()); - registerWindowFunction("rank", new GenericUDAFRank()); - registerWindowFunction("dense_rank", new GenericUDAFDenseRank()); - registerWindowFunction("percent_rank", new GenericUDAFPercentRank()); - registerWindowFunction("cume_dist", new GenericUDAFCumeDist()); - registerWindowFunction("ntile", new GenericUDAFNTile()); - registerWindowFunction("first_value", new GenericUDAFFirstValue()); - registerWindowFunction("last_value", new GenericUDAFLastValue()); - registerWindowFunction(LEAD_FUNC_NAME, new GenericUDAFLead(), false); - registerWindowFunction(LAG_FUNC_NAME, new GenericUDAFLag(), false); - - registerTableFunction(NOOP_TABLE_FUNCTION, NoopResolver.class); - registerTableFunction(NOOP_MAP_TABLE_FUNCTION, NoopWithMapResolver.class); - registerTableFunction(NOOP_STREAMING_TABLE_FUNCTION, NoopStreamingResolver.class); - registerTableFunction(NOOP_STREAMING_MAP_TABLE_FUNCTION, NoopWithMapStreamingResolver.class); - registerTableFunction(WINDOWING_TABLE_FUNCTION, WindowingTableFunctionResolver.class); - registerTableFunction("matchpath", MatchPathResolver.class); - } - - public static void registerTemporaryUDF(String functionName, - Class UDFClass, boolean isOperator) { - registerUDF(false, functionName, UDFClass, isOperator); - } - - static void registerUDF(String functionName, Class UDFClass, - boolean isOperator) { - registerUDF(true, functionName, UDFClass, isOperator); - } - - public static void registerUDF(boolean isNative, String functionName, - Class UDFClass, boolean isOperator) { - registerUDF(isNative, functionName, UDFClass, isOperator, functionName - .toLowerCase()); - } - - public static void registerUDF(String functionName, - Class UDFClass, boolean isOperator, String displayName) { - registerUDF(true, functionName, UDFClass, isOperator, displayName); - } - - public static void registerUDF(boolean isNative, String functionName, - Class UDFClass, boolean isOperator, String displayName) { - if (UDF.class.isAssignableFrom(UDFClass)) { - FunctionInfo fI = new FunctionInfo(isNative, displayName, - new GenericUDFBridge(displayName, isOperator, UDFClass.getName())); - mFunctions.put(functionName.toLowerCase(), fI); - registerNativeStatus(fI); - } else { - throw new RuntimeException("Registering UDF Class " + UDFClass - + " which does not extend " + UDF.class); - } - } - - public static void registerTemporaryGenericUDF(String functionName, - Class genericUDFClass) { - registerGenericUDF(false, functionName, genericUDFClass); - } - - static void registerGenericUDF(String functionName, - Class genericUDFClass) { - registerGenericUDF(true, functionName, genericUDFClass); - } - - public static void registerGenericUDF(boolean isNative, String functionName, - Class genericUDFClass) { - if (GenericUDF.class.isAssignableFrom(genericUDFClass)) { - FunctionInfo fI = new FunctionInfo(isNative, functionName, - (GenericUDF) ReflectionUtils.newInstance(genericUDFClass, null)); - mFunctions.put(functionName.toLowerCase(), fI); - registerNativeStatus(fI); - } else { - throw new RuntimeException("Registering GenericUDF Class " - + genericUDFClass + " which does not extend " + GenericUDF.class); - } - } - - public static void registerTemporaryGenericUDTF(String functionName, - Class genericUDTFClass) { - registerGenericUDTF(false, functionName, genericUDTFClass); - } - - static void registerGenericUDTF(String functionName, - Class genericUDTFClass) { - registerGenericUDTF(true, functionName, genericUDTFClass); - } - - public static void registerGenericUDTF(boolean isNative, String functionName, - Class genericUDTFClass) { - if (GenericUDTF.class.isAssignableFrom(genericUDTFClass)) { - FunctionInfo fI = new FunctionInfo(isNative, functionName, - (GenericUDTF) ReflectionUtils.newInstance(genericUDTFClass, null)); - mFunctions.put(functionName.toLowerCase(), fI); - registerNativeStatus(fI); - } else { - throw new RuntimeException("Registering GenericUDTF Class " - + genericUDTFClass + " which does not extend " + GenericUDTF.class); - } - } - - private static FunctionInfo getFunctionInfoFromMetastore(String functionName) { - FunctionInfo ret = null; - - try { - String dbName; - String fName; - if (FunctionUtils.isQualifiedFunctionName(functionName)) { - String[] parts = FunctionUtils.splitQualifiedFunctionName(functionName); - dbName = parts[0]; - fName = parts[1]; - } else { - // otherwise, qualify using current db - dbName = SessionState.get().getCurrentDatabase().toLowerCase(); - fName = functionName; - } - - // Try looking up function in the metastore - HiveConf conf = SessionState.get().getConf(); - Function func = Hive.get(conf).getFunction(dbName, fName); - if (func != null) { - // Found UDF in metastore - now add it to the function registry - // At this point we should add any relevant jars that would be needed for the UDf. - try { - FunctionTask.addFunctionResources(func.getResourceUris()); - } catch (Exception e) { - LOG.error("Unable to load resources for " + dbName + "." + fName + ":" + e.getMessage(), e); - return null; - } - - Class udfClass = Class.forName(func.getClassName(), true, Utilities.getSessionSpecifiedClassLoader()); - if (registerTemporaryFunction(functionName, udfClass)) { - ret = mFunctions.get(functionName); - } else { - LOG.error(func.getClassName() + " is not a valid UDF class and was not registered."); - } - } - } catch (HiveException e) { - if (!((e.getCause() != null) && (e.getCause() instanceof MetaException)) && - (e.getCause().getCause() != null) && (e.getCause().getCause() instanceof NoSuchObjectException)) { - LOG.info("Unable to lookup UDF in metastore: " + e); - } - } catch (ClassNotFoundException e) { - // Lookup of UDf class failed - LOG.error("Unable to load UDF class: " + e); - } - - return ret; - } - - private static T getQualifiedFunctionInfo( - Map mFunctions, String functionName) { - T functionInfo = mFunctions.get(functionName); - if (functionInfo == null) { - // Try looking up in metastore. - FunctionInfo fi = getFunctionInfoFromMetastore(functionName); - if (fi != null) { - // metastore lookup resulted in function getting added to mFunctions, try again - functionInfo = mFunctions.get(functionName); - } - } - - // HIVE-6672: In HiveServer2 the JARs for this UDF may have been loaded by a different thread, - // and the current thread may not be able to resolve the UDF. Test for this condition - // and if necessary load the JARs in this thread. - if (functionInfo != null) { - loadFunctionResourcesIfNecessary(functionName, functionInfo); - } - - return functionInfo; - } - - private static void checkFunctionClass(CommonFunctionInfo cfi) throws ClassNotFoundException { - // This call will fail for non-generic UDFs using GenericUDFBridge - Class udfClass = cfi.getFunctionClass(); - // Even if we have a reference to the class (which will be the case for GenericUDFs), - // the classloader may not be able to resolve the class, which would mean reflection-based - // methods would fail such as for plan deserialization. Make sure this works too. - Class.forName(udfClass.getName(), true, Utilities.getSessionSpecifiedClassLoader()); - } - - private static void loadFunctionResourcesIfNecessary(String functionName, CommonFunctionInfo cfi) { - try { - // Check if the necessary JARs have been loaded for this function. - checkFunctionClass(cfi); - } catch (Exception e) { - // Unable to resolve the UDF with the classloader. - // Look up the function in the metastore and load any resources. - LOG.debug("Attempting to reload resources for " + functionName); - try { - String[] parts = FunctionUtils.getQualifiedFunctionNameParts(functionName); - HiveConf conf = SessionState.get().getConf(); - Function func = Hive.get(conf).getFunction(parts[0], parts[1]); - if (func != null) { - FunctionTask.addFunctionResources(func.getResourceUris()); - // Check again now that we've loaded the resources in this thread. - checkFunctionClass(cfi); - } else { - // Couldn't find the function .. just rethrow the original error - LOG.error("Unable to reload resources for " + functionName); - throw e; - } - } catch (Exception err) { - throw new RuntimeException(err); - } - } + system.registerGenericUDF(LEAD_FUNC_NAME, GenericUDFLead.class, true); + system.registerGenericUDF(LAG_FUNC_NAME, GenericUDFLag.class, true); + + system.registerGenericUDAF("row_number", new GenericUDAFRowNumber(), true); + system.registerGenericUDAF("rank", new GenericUDAFRank(), true); + system.registerGenericUDAF("dense_rank", new GenericUDAFDenseRank(), true); + system.registerGenericUDAF("percent_rank", new GenericUDAFPercentRank(), true); + system.registerGenericUDAF("cume_dist", new GenericUDAFCumeDist(), true); + system.registerGenericUDAF("ntile", new GenericUDAFNTile(), true); + system.registerGenericUDAF("first_value", new GenericUDAFFirstValue(), true); + system.registerGenericUDAF("last_value", new GenericUDAFLastValue(), true); + system.registerWindowFunction(LEAD_FUNC_NAME, new GenericUDAFLead(), true); + system.registerWindowFunction(LAG_FUNC_NAME, new GenericUDAFLag(), true); + + system.registerTableFunction(NOOP_TABLE_FUNCTION, NoopResolver.class); + system.registerTableFunction(NOOP_MAP_TABLE_FUNCTION, NoopWithMapResolver.class); + system.registerTableFunction(NOOP_STREAMING_TABLE_FUNCTION, NoopStreamingResolver.class); + system.registerTableFunction(NOOP_STREAMING_MAP_TABLE_FUNCTION, NoopWithMapStreamingResolver.class); + system.registerTableFunction(WINDOWING_TABLE_FUNCTION, WindowingTableFunctionResolver.class); + system.registerTableFunction(MATCH_PATH_TABLE_FUNCTION, MatchPathResolver.class); } public static String getNormalizedFunctionName(String fn) { // Does the same thing as getFunctionInfo, except for getting the function info. fn = fn.toLowerCase(); - return (FunctionUtils.isQualifiedFunctionName(fn) || mFunctions.get(fn) != null) ? fn + return (FunctionUtils.isQualifiedFunctionName(fn) || getFunctionInfo(fn) != null) ? fn : FunctionUtils.qualifyFunctionName( fn, SessionState.get().getCurrentDatabase().toLowerCase()); } - private static T getFunctionInfo( - Map mFunctions, String functionName) { - functionName = functionName.toLowerCase(); - T functionInfo = null; - if (FunctionUtils.isQualifiedFunctionName(functionName)) { - functionInfo = getQualifiedFunctionInfo(mFunctions, functionName); - } else { - // First try without qualifiers - would resolve builtin/temp functions. - // Otherwise try qualifying with current db name. - functionInfo = mFunctions.get(functionName); - if (functionInfo == null && !FunctionUtils.isQualifiedFunctionName(functionName)) { - String qualifiedName = FunctionUtils.qualifyFunctionName(functionName, - SessionState.get().getCurrentDatabase().toLowerCase()); - functionInfo = getQualifiedFunctionInfo(mFunctions, qualifiedName); - } - } - return functionInfo; - } - public static FunctionInfo getFunctionInfo(String functionName) { - return getFunctionInfo(mFunctions, functionName); + FunctionInfo info = getTemporaryFunctionInfo(functionName); + return info != null ? info : system.getFunctionInfo(functionName); } - /** - * Returns a set of registered function names. This is used for the CLI - * command "SHOW FUNCTIONS;" - * - * @return set of strings contains function names - */ - public static Set getFunctionNames() { - return getFunctionNames(true); + public static FunctionInfo getTemporaryFunctionInfo(String functionName) { + Registry registry = SessionState.getRegistry(); + return registry == null ? null : registry.getFunctionInfo(functionName); } - private static Set getFunctionNames(boolean searchMetastore) { - Set functionNames = mFunctions.keySet(); - if (searchMetastore) { - functionNames = new HashSet(functionNames); - try { - Hive db = getHive(); - List dbNames = db.getAllDatabases(); - - for (String dbName : dbNames) { - List funcNames = db.getFunctions(dbName, "*"); - for (String funcName : funcNames) { - functionNames.add(FunctionUtils.qualifyFunctionName(funcName, dbName)); - } - } - } catch (Exception e) { - LOG.error(e); - // Continue on, we can still return the functions we've gotten to this point. - } - } - return functionNames; + public static WindowFunctionInfo getWindowFunctionInfo(String functionName) { + Registry registry = SessionState.getRegistry(); + WindowFunctionInfo info = registry == null ? null : registry.getWindowFunctionInfo(functionName); + return info != null ? info : system.getWindowFunctionInfo(functionName); } - public static Hive getHive() throws HiveException { - return Hive.get(SessionState.get().getConf()); + public static Set getFunctionNames() { + Set names = new TreeSet(); + names.addAll(system.getFunctionNames(true)); + if (SessionState.getRegistry() != null) { + names.addAll(SessionState.getRegistry().getCurrentFunctionNames()); + } + return names; } - /** - * Returns a set of registered function names. This is used for the CLI - * command "SHOW FUNCTIONS 'regular expression';" Returns an empty set when - * the regular expression is not valid. - * - * @param funcPatternStr - * regular expression of the interested function names - * @return set of strings contains function names - */ public static Set getFunctionNames(String funcPatternStr) { Set funcNames = new TreeSet(); - Pattern funcPattern = null; - try { - funcPattern = Pattern.compile(funcPatternStr); - } catch (PatternSyntaxException e) { - return funcNames; - } - for (String funcName : mFunctions.keySet()) { - if (funcPattern.matcher(funcName).matches()) { - funcNames.add(funcName); - } + funcNames.addAll(system.getFunctionNames(funcPatternStr)); + if (SessionState.getRegistry() != null) { + funcNames.addAll(SessionState.getRegistry().getFunctionNames(funcPatternStr)); } return funcNames; } @@ -741,7 +480,7 @@ public static Hive getHive() throws HiveException { */ public static Set getFunctionNamesByLikePattern(String funcPatternStr) { Set funcNames = new TreeSet(); - Set allFuncs = getFunctionNames(true); + Set allFuncs = getFunctionNames(); String[] subpatterns = funcPatternStr.trim().split("\\|"); for (String subpattern : subpatterns) { subpattern = "(?i)" + subpattern.replaceAll("\\*", ".*"); @@ -762,28 +501,19 @@ public static Hive getHive() throws HiveException { /** * Returns the set of synonyms of the supplied function. * - * @param funcName - * the name of the function + * @param funcName the name of the function * @return Set of synonyms for funcName */ public static Set getFunctionSynonyms(String funcName) { Set synonyms = new HashSet(); - FunctionInfo funcInfo = getFunctionInfo(funcName); if (null == funcInfo) { return synonyms; } - - Class funcClass = funcInfo.getFunctionClass(); - for (String name : mFunctions.keySet()) { - if (name.equals(funcName)) { - continue; - } - if (mFunctions.get(name).getFunctionClass().equals(funcClass)) { - synonyms.add(name); - } + system.getFunctionSynonyms(funcName, funcInfo, synonyms); + if (SessionState.getRegistry() != null) { + SessionState.getRegistry().getFunctionSynonyms(funcName, funcInfo, synonyms); } - return synonyms; } @@ -1012,7 +742,7 @@ public static PrimitiveCategory getCommonCategory(TypeInfo a, TypeInfo b) { // If either is not a numeric type, return null. return null; } - + return (ai > bi) ? pcA : pcB; } @@ -1146,20 +876,11 @@ public static GenericUDAFEvaluator getGenericUDAFEvaluator(String name, public static GenericUDAFEvaluator getGenericWindowingEvaluator(String name, List argumentOIs, boolean isDistinct, boolean isAllColumns) throws SemanticException { - - WindowFunctionInfo finfo = windowFunctions.get(name.toLowerCase()); - if (finfo == null) { return null;} - if ( !name.toLowerCase().equals(LEAD_FUNC_NAME) && - !name.toLowerCase().equals(LAG_FUNC_NAME) ) { - return getGenericUDAFEvaluator(name, argumentOIs, isDistinct, isAllColumns); - } - - // this must be lead/lag UDAF - ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; - GenericUDAFResolver udafResolver = finfo.getfInfo().getGenericUDAFResolver(); - GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( - argumentOIs.toArray(args), isDistinct, isAllColumns); - return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); + Registry registry = SessionState.getRegistry(); + GenericUDAFEvaluator evaluator = registry == null ? null : + registry.getGenericWindowingEvaluator(name, argumentOIs, isDistinct, isAllColumns); + return evaluator != null ? evaluator : + system.getGenericWindowingEvaluator(name, argumentOIs, isDistinct, isAllColumns); } /** @@ -1183,61 +904,6 @@ public static GenericUDAFEvaluator getGenericWindowingEvaluator(String name, return getMethodInternal(udfClass, mlist, exact, argumentClasses); } - public static void registerTemporaryGenericUDAF(String functionName, - GenericUDAFResolver genericUDAFResolver) { - registerGenericUDAF(false, functionName, genericUDAFResolver); - } - - static void registerGenericUDAF(String functionName, - GenericUDAFResolver genericUDAFResolver) { - registerGenericUDAF(true, functionName, genericUDAFResolver); - } - - public static void registerGenericUDAF(boolean isNative, String functionName, - GenericUDAFResolver genericUDAFResolver) { - FunctionInfo fi = new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver); - mFunctions.put(functionName.toLowerCase(), fi); - - // All aggregate functions should also be usable as window functions - addFunctionInfoToWindowFunctions(functionName, fi); - - registerNativeStatus(fi); - } - - public static void registerTemporaryUDAF(String functionName, - Class udafClass) { - registerUDAF(false, functionName, udafClass); - } - - static void registerUDAF(String functionName, Class udafClass) { - registerUDAF(true, functionName, udafClass); - } - - public static void registerUDAF(boolean isNative, String functionName, - Class udafClass) { - FunctionInfo fi = new FunctionInfo(isNative, - functionName.toLowerCase(), new GenericUDAFBridge( - (UDAF) ReflectionUtils.newInstance(udafClass, null))); - mFunctions.put(functionName.toLowerCase(), fi); - - // All aggregate functions should also be usable as window functions - addFunctionInfoToWindowFunctions(functionName, fi); - - registerNativeStatus(fi); - } - - public static void unregisterTemporaryUDF(String functionName) throws HiveException { - FunctionInfo fi = mFunctions.get(functionName.toLowerCase()); - if (fi != null) { - if (!fi.isNative()) { - mFunctions.remove(functionName.toLowerCase()); - } else { - throw new HiveException("Function " + functionName - + " is hive native, it can't be dropped"); - } - } - } - public static GenericUDAFResolver getGenericUDAFResolver(String functionName) { if (LOG.isDebugEnabled()) { LOG.debug("Looking up GenericUDAF: " + functionName); @@ -1565,8 +1231,7 @@ public static GenericUDF cloneGenericUDF(GenericUDF genericUDF) { clonedUDF = new GenericUDFMacro(bridge.getMacroName(), bridge.getBody(), bridge.getColNames(), bridge.getColTypes()); } else { - clonedUDF = (GenericUDF) ReflectionUtils - .newInstance(genericUDF.getClass(), null); + clonedUDF = ReflectionUtils.newInstance(genericUDF.getClass(), null); } if (clonedUDF != null) { @@ -1604,32 +1269,23 @@ public static GenericUDTF cloneGenericUDTF(GenericUDTF genericUDTF) { if (null == genericUDTF) { return null; } - return (GenericUDTF) ReflectionUtils.newInstance(genericUDTF.getClass(), - null); + return ReflectionUtils.newInstance(genericUDTF.getClass(), null); } /** * Get the UDF class from an exprNodeDesc. Returns null if the exprNodeDesc * does not contain a UDF class. */ - private static Class getGenericUDFClassFromExprDesc(ExprNodeDesc desc) { + private static Class getUDFClassFromExprDesc(ExprNodeDesc desc) { if (!(desc instanceof ExprNodeGenericFuncDesc)) { return null; } ExprNodeGenericFuncDesc genericFuncDesc = (ExprNodeGenericFuncDesc) desc; - return genericFuncDesc.getGenericUDF().getClass(); - } - - /** - * Get the UDF class from an exprNodeDesc. Returns null if the exprNodeDesc - * does not contain a UDF class. - */ - private static Class getUDFClassFromExprDesc(ExprNodeDesc desc) { - if (!(desc instanceof ExprNodeGenericFuncDesc)) { - return null; + GenericUDF genericUDF = genericFuncDesc.getGenericUDF(); + if (genericUDF instanceof GenericUDFBridge) { + return ((GenericUDFBridge) genericUDF).getUdfClass(); } - ExprNodeGenericFuncDesc genericFuncDesc = (ExprNodeGenericFuncDesc) desc; - return genericFuncDesc.getGenericUDF().getClass(); + return genericUDF.getClass(); } /** @@ -1691,7 +1347,7 @@ public static boolean isStateful(GenericUDF genericUDF) { * Returns whether the exprNodeDesc is a node of "and", "or", "not". */ public static boolean isOpAndOrNot(ExprNodeDesc desc) { - Class genericUdfClass = getGenericUDFClassFromExprDesc(desc); + Class genericUdfClass = getUDFClassFromExprDesc(desc); return GenericUDFOPAnd.class == genericUdfClass || GenericUDFOPOr.class == genericUdfClass || GenericUDFOPNot.class == genericUdfClass; @@ -1701,21 +1357,21 @@ public static boolean isOpAndOrNot(ExprNodeDesc desc) { * Returns whether the exprNodeDesc is a node of "and". */ public static boolean isOpAnd(ExprNodeDesc desc) { - return GenericUDFOPAnd.class == getGenericUDFClassFromExprDesc(desc); + return GenericUDFOPAnd.class == getUDFClassFromExprDesc(desc); } /** * Returns whether the exprNodeDesc is a node of "or". */ public static boolean isOpOr(ExprNodeDesc desc) { - return GenericUDFOPOr.class == getGenericUDFClassFromExprDesc(desc); + return GenericUDFOPOr.class == getUDFClassFromExprDesc(desc); } /** * Returns whether the exprNodeDesc is a node of "not". */ public static boolean isOpNot(ExprNodeDesc desc) { - return GenericUDFOPNot.class == getGenericUDFClassFromExprDesc(desc); + return GenericUDFOPNot.class == getUDFClassFromExprDesc(desc); } /** @@ -1767,45 +1423,16 @@ public static boolean isOpPreserveInputName(ExprNodeDesc desc) { * @return true if udfClass's type was recognized (so registration * succeeded); false otherwise */ - public static boolean registerTemporaryFunction( - String functionName, Class udfClass) { - - UDFClassType udfClassType = FunctionUtils.getUDFClassType(udfClass); - switch (udfClassType) { - case UDF: - FunctionRegistry.registerTemporaryUDF( - functionName, (Class) udfClass, false); - break; - case GENERIC_UDF: - FunctionRegistry.registerTemporaryGenericUDF( - functionName, (Class) udfClass); - break; - case GENERIC_UDTF: - FunctionRegistry.registerTemporaryGenericUDTF( - functionName, (Class) udfClass); - break; - case UDAF: - FunctionRegistry.registerTemporaryUDAF( - functionName, (Class) udfClass); - break; - case GENERIC_UDAF_RESOLVER: - FunctionRegistry.registerTemporaryGenericUDAF( - functionName, (GenericUDAFResolver) - ReflectionUtils.newInstance(udfClass, null)); - break; - case TABLE_FUNCTION_RESOLVER: - FunctionRegistry.registerTableFunction( - functionName, (Class)udfClass); - break; - default: - return false; - } - return true; + public static boolean registerTemporaryFunction(String functionName, Class udfClass) { + return SessionState.getRegistryForWrite().registerFunction(functionName, udfClass, false) != null; + } + public static void unregisterTemporaryFunction(String functionName) throws HiveException { + SessionState.getRegistryForWrite().unregisterFunction(functionName); } /** - * Registers thae appropriate kind of temporary function based on a class's + * Registers the appropriate kind of temporary function based on a class's * type. * * @param macroName name under which to register the macro @@ -1817,53 +1444,16 @@ public static boolean registerTemporaryFunction( * @param colTypes the types of the arguments to the macro */ public static void registerTemporaryMacro( - String macroName, ExprNodeDesc body, - List colNames, List colTypes) { - - FunctionInfo fI = new FunctionInfo(false, macroName, - new GenericUDFMacro(macroName, body, colNames, colTypes)); - mFunctions.put(macroName.toLowerCase(), fI); - registerNativeStatus(fI); + String macroName, ExprNodeDesc body, List colNames, List colTypes) { + SessionState.getRegistryForWrite().registerMacro(macroName, body, colNames, colTypes); } - /** - * Registers Hive functions from a plugin jar, using metadata from - * the jar's META-INF/class-info.xml. - * - * @param jarLocation URL for reading jar file - * - * @param classLoader classloader to use for loading function classes - */ - public static void registerFunctionsFromPluginJar( - URL jarLocation, - ClassLoader classLoader) throws Exception { + public static boolean registerPermanentFunction(String functionName, Class udfClass) { + return system.registerFunction(functionName, udfClass, false) != null; + } - URL url = new URL("jar:" + jarLocation + "!/META-INF/class-info.xml"); - InputStream inputStream = null; - try { - inputStream = url.openStream(); - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = dbf.newDocumentBuilder(); - Document doc = docBuilder.parse(inputStream); - Element root = doc.getDocumentElement(); - if (!root.getTagName().equals("ClassList")) { - return; - } - NodeList children = root.getElementsByTagName("Class"); - for (int i = 0; i < children.getLength(); ++i) { - Element child = (Element) children.item(i); - String javaName = child.getAttribute("javaname"); - String sqlName = child.getAttribute("sqlname"); - Class udfClass = Class.forName(javaName, true, classLoader); - boolean registered = registerTemporaryFunction(sqlName, udfClass); - if (!registered) { - throw new RuntimeException( - "Class " + udfClass + " is not a Hive function implementation"); - } - } - } finally { - IOUtils.closeStream(inputStream); - } + public static void unregisterPermanentFunction(String functionName) throws HiveException { + system.unregisterFunction(functionName); } private FunctionRegistry() { @@ -1873,42 +1463,6 @@ private FunctionRegistry() { //---------PTF functions------------ - public static void registerWindowFunction(String name, GenericUDAFResolver wFn) - { - registerWindowFunction(name, wFn, true); - } - - /** - * Typically a WindowFunction is the same as a UDAF. The only exceptions are Lead & Lag UDAFs. These - * are not registered as regular UDAFs because - * - we plan to support Lead & Lag as UDFs (usable only within argument expressions - * of UDAFs when windowing is involved). Since mFunctions holds both UDFs and UDAFs we cannot - * add both FunctionInfos to mFunctions. - * We choose to only register UDFs in mFunctions. The implication of this is that Lead/Lag UDAFs - * are only usable when windowing is involved. - * - * @param name - * @param wFn - * @param registerAsUDAF - */ - public static void registerWindowFunction(String name, GenericUDAFResolver wFn, boolean registerAsUDAF) - { - FunctionInfo fInfo = null; - if (registerAsUDAF) { - // Just register the function normally, will also get added to window functions. - registerGenericUDAF(true, name, wFn); - } - else { - name = name.toLowerCase(); - fInfo = new FunctionInfo(true, name, wFn); - addFunctionInfoToWindowFunctions(name, fInfo); - } - } - - public static WindowFunctionInfo getWindowFunctionInfo(String functionName) { - return getFunctionInfo(windowFunctions, functionName); - } - /** * Both UDF and UDAF functions can imply order for analytical functions * @@ -1920,13 +1474,11 @@ public static WindowFunctionInfo getWindowFunctionInfo(String functionName) { public static boolean impliesOrder(String functionName) { FunctionInfo info = getFunctionInfo(functionName); - if (info != null) { - if (info.isGenericUDF()) { - UDFType type = - AnnotationUtils.getAnnotation(info.getGenericUDF().getClass(), UDFType.class); - if (type != null) { - return type.impliesOrder(); - } + if (info != null && info.isGenericUDF()) { + UDFType type = + AnnotationUtils.getAnnotation(info.getGenericUDF().getClass(), UDFType.class); + if (type != null) { + return type.impliesOrder(); } } WindowFunctionInfo windowInfo = getWindowFunctionInfo(functionName); @@ -1936,33 +1488,23 @@ public static boolean impliesOrder(String functionName) { return false; } - static private void addFunctionInfoToWindowFunctions(String functionName, - FunctionInfo functionInfo) { - // Assumes that the caller has already verified that functionInfo is for an aggregate function - WindowFunctionInfo wInfo = new WindowFunctionInfo(functionInfo); - windowFunctions.put(functionName.toLowerCase(), wInfo); - } - - public static boolean isTableFunction(String name) - { - FunctionInfo tFInfo = getFunctionInfo(name); + public static boolean isTableFunction(String functionName) { + FunctionInfo tFInfo = getFunctionInfo(functionName); return tFInfo != null && !tFInfo.isInternalTableFunction() && tFInfo.isTableFunction(); } - public static TableFunctionResolver getTableFunctionResolver(String name) - { - FunctionInfo tfInfo = getFunctionInfo(name); - if(tfInfo.isTableFunction()) { + public static TableFunctionResolver getTableFunctionResolver(String functionName) { + FunctionInfo tfInfo = getFunctionInfo(functionName); + if (tfInfo != null && tfInfo.isTableFunction()) { return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionClass(), null); } return null; } - public static TableFunctionResolver getWindowingTableFunction() - { + public static TableFunctionResolver getWindowingTableFunction() { return getTableFunctionResolver(WINDOWING_TABLE_FUNCTION); } - + public static boolean isNoopFunction(String fnName) { fnName = fnName.toLowerCase(); return fnName.equals(NOOP_MAP_TABLE_FUNCTION) || @@ -1971,13 +1513,6 @@ public static boolean isNoopFunction(String fnName) { fnName.equals(NOOP_STREAMING_TABLE_FUNCTION); } - public static void registerTableFunction(String name, Class tFnCls) - { - FunctionInfo tInfo = new FunctionInfo(name, tFnCls); - mFunctions.put(name.toLowerCase(), tInfo); - registerNativeStatus(tInfo); - } - /** * Use this to check if function is ranking function * @@ -1986,10 +1521,10 @@ public static void registerTableFunction(String name, Class udfClass = getUDFClassFromExprDesc(fnExpr); - if (udfClass == null) { - udfClass = getGenericUDFClassFromExprDesc(fnExpr); + Class udfClass = FunctionRegistry.getUDFClassFromExprDesc(fnExpr); + if (udfClass != null) { + Registry registry = SessionState.getRegistry(); + return registry != null ? registry.isNativeFunc(udfClass) : system.isNativeFunc(udfClass); } - return nativeUdfs.contains(udfClass); - } - - private static void registerNativeStatus(FunctionInfo fi) { - if (!fi.isNative()) { - return; - } - nativeUdfs.add(fi.getFunctionClass()); + return false; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java index 569c125..4141e3b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java @@ -135,8 +135,7 @@ private int createPermanentFunction(Hive db, CreateFunctionDesc createFunctionDe return 1; } - // TODO: There should be a registerPermanentFunction() - addedToRegistry = FunctionRegistry.registerTemporaryFunction(registeredName, udfClass); + addedToRegistry = FunctionRegistry.registerPermanentFunction(registeredName, udfClass); if (!addedToRegistry) { console.printError("Failed to register " + registeredName + " using class " + createFunctionDesc.getClassName()); @@ -160,7 +159,7 @@ private int createPermanentFunction(Hive db, CreateFunctionDesc createFunctionDe console.printError("FAILED: Class " + createFunctionDesc.getClassName() + " not found"); LOG.info("create function: " + StringUtils.stringifyException(e)); if (addedToRegistry) { - FunctionRegistry.unregisterTemporaryUDF(registeredName); + FunctionRegistry.unregisterPermanentFunction(registeredName); } return 1; } @@ -204,7 +203,7 @@ private int createMacro(CreateMacroDesc createMacroDesc) { private int dropMacro(DropMacroDesc dropMacroDesc) { try { - FunctionRegistry.unregisterTemporaryUDF(dropMacroDesc + FunctionRegistry.unregisterTemporaryFunction(dropMacroDesc .getMacroName()); return 0; } catch (HiveException e) { @@ -221,8 +220,7 @@ private int dropPermanentFunction(Hive db, DropFunctionDesc dropFunctionDesc) { String funcName = qualifiedNameParts[1]; String registeredName = FunctionUtils.qualifyFunctionName(funcName, dbName); - // TODO: there should be a unregisterPermanentUDF() - FunctionRegistry.unregisterTemporaryUDF(registeredName); + FunctionRegistry.unregisterPermanentFunction(registeredName); db.dropFunction(dbName, funcName); return 0; @@ -235,8 +233,7 @@ private int dropPermanentFunction(Hive db, DropFunctionDesc dropFunctionDesc) { private int dropTemporaryFunction(DropFunctionDesc dropFunctionDesc) { try { - FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc - .getFunctionName()); + FunctionRegistry.unregisterTemporaryFunction(dropFunctionDesc.getFunctionName()); return 0; } catch (HiveException e) { LOG.info("drop function: " + StringUtils.stringifyException(e)); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java new file mode 100644 index 0000000..171b10b --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java @@ -0,0 +1,530 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver2; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMacro; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; +import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo; +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.util.ReflectionUtils; + +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +// Extracted from FunctionRegistry +public class Registry { + + private static final Log LOG = LogFactory.getLog(Registry.class); + + // prefix to discern LEAD/LAG UDFs from window functions with the same name + private static final String WINDOW_FUNC_PREFIX = "@_"; + + /** + * The mapping from expression function names to expression classes. + */ + private final Map mFunctions = new LinkedHashMap(); + private final Set> nativeUdfs = Collections.synchronizedSet(new HashSet>()); + + private final boolean isNative; + + Registry(boolean isNative) { + this.isNative = isNative; + } + + public Registry() { + this(false); + } + + /** + * Registers the appropriate kind of temporary function based on a class's + * type. + * + * @param functionName name under which to register function + * @param udfClass class implementing UD[A|T]F + * @return true if udfClass's type was recognized (so registration + * succeeded); false otherwise + */ + @SuppressWarnings("unchecked") + public FunctionInfo registerFunction( + String functionName, Class udfClass, boolean isNative) { + + FunctionUtils.UDFClassType udfClassType = FunctionUtils.getUDFClassType(udfClass); + switch (udfClassType) { + case UDF: + return registerUDF( + functionName, (Class) udfClass, false, isNative); + case GENERIC_UDF: + return registerGenericUDF( + functionName, (Class) udfClass, isNative); + case GENERIC_UDTF: + return registerGenericUDTF( + functionName, (Class) udfClass, isNative); + case UDAF: + return registerUDAF( + functionName, (Class) udfClass, isNative); + case GENERIC_UDAF_RESOLVER: + return registerGenericUDAF( + functionName, (GenericUDAFResolver) + ReflectionUtils.newInstance(udfClass, null), isNative); + case TABLE_FUNCTION_RESOLVER: + return registerTableFunction( + functionName, (Class) udfClass); + } + return null; + + } + + FunctionInfo registerUDF(String functionName, + Class UDFClass, boolean isOperator, boolean isNative) { + return registerUDF(functionName, UDFClass, isOperator, functionName.toLowerCase(), isNative); + } + + FunctionInfo registerUDF(String functionName, + Class UDFClass, boolean isOperator, String displayName, boolean isNative) { + validateClass(UDFClass, UDF.class); + FunctionInfo fI = new FunctionInfo(isNative, displayName, + new GenericUDFBridge(displayName, isOperator, UDFClass.getName())); + addFunction(functionName, fI); + return fI; + } + + FunctionInfo registerGenericUDF(String functionName, + Class genericUDFClass, boolean isNative) { + validateClass(genericUDFClass, GenericUDF.class); + FunctionInfo fI = new FunctionInfo(isNative, functionName, + ReflectionUtils.newInstance(genericUDFClass, null)); + addFunction(functionName, fI); + return fI; + } + + FunctionInfo registerGenericUDTF(String functionName, + Class genericUDTFClass, boolean isNative) { + validateClass(genericUDTFClass, GenericUDTF.class); + FunctionInfo fI = new FunctionInfo(isNative, functionName, + ReflectionUtils.newInstance(genericUDTFClass, null)); + addFunction(functionName, fI); + return fI; + } + + FunctionInfo registerGenericUDAF(String functionName, + GenericUDAFResolver genericUDAFResolver, boolean isNative) { + FunctionInfo function = new WindowFunctionInfo(isNative, functionName, genericUDAFResolver); + addFunction(functionName, function); + addFunction(WINDOW_FUNC_PREFIX + functionName, function); + return function; + } + + FunctionInfo registerUDAF(String functionName, Class udafClass, + boolean isNative) { + validateClass(udafClass, UDAF.class); + FunctionInfo function = new WindowFunctionInfo(isNative, functionName, + new GenericUDAFBridge(ReflectionUtils.newInstance(udafClass, null))); + addFunction(functionName, function); + addFunction(WINDOW_FUNC_PREFIX + functionName, function); + return function; + } + + FunctionInfo registerTableFunction(String functionName, + Class tFnCls) { + validateClass(tFnCls, TableFunctionResolver.class); + FunctionInfo function = new FunctionInfo(functionName, tFnCls); + addFunction(functionName, function); + return function; + } + + FunctionInfo registerMacro(String macroName, ExprNodeDesc body, + List colNames, List colTypes) { + FunctionInfo fI = new FunctionInfo(false, macroName, + new GenericUDFMacro(macroName, body, colNames, colTypes)); + addFunction(macroName, fI); + return fI; + } + + /** + * Typically a WindowFunction is the same as a UDAF. The only exceptions are Lead & Lag UDAFs. These + * are not registered as regular UDAFs because + * - we plan to support Lead & Lag as UDFs (usable only within argument expressions + * of UDAFs when windowing is involved). Since mFunctions holds both UDFs and UDAFs we cannot + * add both FunctionInfos to mFunctions. + * We choose to only register UDFs in mFunctions. The implication of this is that Lead/Lag UDAFs + * are only usable when windowing is involved. + * + * @param name + * @param wFn + * @param registerAsUDAF + */ + void registerWindowFunction(String name, GenericUDAFResolver wFn, boolean isNative) { + addFunction(WINDOW_FUNC_PREFIX + name, new WindowFunctionInfo(isNative, name, wFn)); + } + + private void validateClass(Class input, Class expected) { + if (!expected.isAssignableFrom(input)) { + throw new RuntimeException("Registering UDF Class " + input + + " which does not extend " + expected); + } + } + + /** + * Looks up the function name in the registry. If enabled, will attempt to search the metastore + * for the function. + * @param functionName + * @return + */ + public synchronized FunctionInfo getFunctionInfo(String functionName) { + functionName = functionName.toLowerCase(); + if (FunctionUtils.isQualifiedFunctionName(functionName)) { + return getQualifiedFunctionInfo(functionName); + } + // First try without qualifiers - would resolve builtin/temp functions. + // Otherwise try qualifying with current db name. + FunctionInfo functionInfo = mFunctions.get(functionName); + if (functionInfo == null) { + String qualifiedName = FunctionUtils.qualifyFunctionName( + functionName, SessionState.get().getCurrentDatabase().toLowerCase()); + functionInfo = getQualifiedFunctionInfo(qualifiedName); + } + return functionInfo; + } + + public WindowFunctionInfo getWindowFunctionInfo(String functionName) { + FunctionInfo info = getFunctionInfo(WINDOW_FUNC_PREFIX + functionName); + if (info instanceof WindowFunctionInfo) { + return (WindowFunctionInfo) info; + } + return null; + } + + /** + * @param fnExpr Function expression. + * @return True iff the fnExpr represents a hive built-in function. + */ + public boolean isNativeFunc(Class udfClass) { + return nativeUdfs.contains(udfClass); + } + + public synchronized Set getCurrentFunctionNames() { + return mFunctions.keySet(); + } + + /** + * Returns a set of registered function names. This is used for the CLI + * command "SHOW FUNCTIONS;" + * + * @return set of strings contains function names + */ + public Set getFunctionNames(boolean searchMetastore) { + Set functionNames = getCurrentFunctionNames(); + if (searchMetastore) { + functionNames = new HashSet(functionNames); + try { + Hive db = Hive.get(); + List dbNames = db.getAllDatabases(); + + for (String dbName : dbNames) { + List funcNames = db.getFunctions(dbName, "*"); + for (String funcName : funcNames) { + functionNames.add(FunctionUtils.qualifyFunctionName(funcName, dbName)); + } + } + } catch (Exception e) { + LOG.error(e); + // Continue on, we can still return the functions we've gotten to this point. + } + } + return functionNames; + } + + /** + * Returns a set of registered function names. This is used for the CLI + * command "SHOW FUNCTIONS 'regular expression';" Returns an empty set when + * the regular expression is not valid. + * + * @param funcPatternStr regular expression of the interested function names + * @return set of strings contains function names + */ + public synchronized Set getFunctionNames(String funcPatternStr) { + Set funcNames = new TreeSet(); + Pattern funcPattern; + try { + funcPattern = Pattern.compile(funcPatternStr); + } catch (PatternSyntaxException e) { + return funcNames; + } + for (String funcName : mFunctions.keySet()) { + if (funcPattern.matcher(funcName).matches()) { + funcNames.add(funcName); + } + } + return funcNames; + } + + /** + * Adds to the set of synonyms of the supplied function. + * @param funcName + * @param funcInfo + * @param synonyms + */ + public synchronized void getFunctionSynonyms(String funcName, FunctionInfo funcInfo, Set synonyms) { + Class funcClass = funcInfo.getFunctionClass(); + for (String name : mFunctions.keySet()) { + if (name.equals(funcName)) { + continue; + } + if (mFunctions.get(name).getFunctionClass().equals(funcClass)) { + synonyms.add(name); + } + } + } + + /** + * Get the GenericUDAF evaluator for the name and argumentClasses. + * + * @param name the name of the UDAF + * @param argumentOIs + * @param isDistinct + * @param isAllColumns + * @return The UDAF evaluator + */ + @SuppressWarnings("deprecation") + public GenericUDAFEvaluator getGenericUDAFEvaluator(String name, + List argumentOIs, boolean isDistinct, + boolean isAllColumns) throws SemanticException { + + GenericUDAFResolver udafResolver = getGenericUDAFResolver(name); + if (udafResolver == null) { + return null; + } + + GenericUDAFEvaluator udafEvaluator; + ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; + // Can't use toArray here because Java is dumb when it comes to + // generics + arrays. + for (int ii = 0; ii < argumentOIs.size(); ++ii) { + args[ii] = argumentOIs.get(ii); + } + + GenericUDAFParameterInfo paramInfo = + new SimpleGenericUDAFParameterInfo( + args, isDistinct, isAllColumns); + if (udafResolver instanceof GenericUDAFResolver2) { + udafEvaluator = + ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); + } else { + udafEvaluator = udafResolver.getEvaluator(paramInfo.getParameters()); + } + return udafEvaluator; + } + + public GenericUDAFEvaluator getGenericWindowingEvaluator(String functionName, + List argumentOIs, boolean isDistinct, boolean isAllColumns) + throws SemanticException { + functionName = functionName.toLowerCase(); + WindowFunctionInfo info = getWindowFunctionInfo(functionName); + if (info == null) { + return null; + } + if (!functionName.equals(FunctionRegistry.LEAD_FUNC_NAME) && + !functionName.equals(FunctionRegistry.LAG_FUNC_NAME)) { + return getGenericUDAFEvaluator(functionName, argumentOIs, isDistinct, isAllColumns); + } + + // this must be lead/lag UDAF + ObjectInspector args[] = new ObjectInspector[argumentOIs.size()]; + GenericUDAFResolver udafResolver = info.getGenericUDAFResolver(); + GenericUDAFParameterInfo paramInfo = new SimpleGenericUDAFParameterInfo( + argumentOIs.toArray(args), isDistinct, isAllColumns); + return ((GenericUDAFResolver2) udafResolver).getEvaluator(paramInfo); + } + + private synchronized void addFunction(String functionName, FunctionInfo function) { + functionName = functionName.toLowerCase(); + FunctionInfo prev = mFunctions.get(functionName); + if (prev != null && prev.isNative() && !function.isNative()) { + throw new RuntimeException("Function " + functionName + " is hive native, " + + "which cannot be overriden by non-native function."); + } + mFunctions.put(functionName, function); + if (function.isNative()) { + nativeUdfs.add(function.getFunctionClass()); + } + } + + public synchronized void unregisterFunction(String functionName) throws HiveException { + functionName = functionName.toLowerCase(); + FunctionInfo fi = mFunctions.get(functionName); + if (fi != null) { + if (fi.isNative()) { + throw new HiveException(ErrorMsg.DROP_NATIVE_FUNCTION.getMsg(functionName)); + } + mFunctions.remove(functionName); + nativeUdfs.remove(fi.getFunctionClass()); + } + } + + public GenericUDAFResolver getGenericUDAFResolver(String functionName) { + FunctionInfo info = getFunctionInfo(functionName); + if (info != null) { + return info.getGenericUDAFResolver(); + } + return null; + } + + private FunctionInfo getQualifiedFunctionInfo(String functionName) { + FunctionInfo info = mFunctions.get(functionName); + + if (!canLookupMetaStoreFunctions()) { + return info; + } + + if (info == null) { + // Try looking up in metastore. + info = registerFunctionInMetastore(functionName); + } + // HIVE-6672: In HiveServer2 the JARs for this UDF may have been loaded by a different thread, + // and the current thread may not be able to resolve the UDF. Test for this condition + // and if necessary load the JARs in this thread. + if (info != null) { + loadFunctionResourcesIfNecessary(functionName, info); + } + return info; + } + + private FunctionInfo registerFunctionInMetastore(String functionName) { + FunctionInfo ret = null; + + try { + String dbName; + String fName; + if (FunctionUtils.isQualifiedFunctionName(functionName)) { + String[] parts = FunctionUtils.splitQualifiedFunctionName(functionName); + dbName = parts[0]; + fName = parts[1]; + } else { + // otherwise, qualify using current db + dbName = SessionState.get().getCurrentDatabase().toLowerCase(); + fName = functionName; + } + + // Try looking up function in the metastore + Function func = Hive.get().getFunction(dbName, fName); + if (func != null) { + // Found UDF in metastore - now add it to the function registry + // At this point we should add any relevant jars that would be needed for the UDf. + try { + FunctionTask.addFunctionResources(func.getResourceUris()); + } catch (Exception e) { + LOG.error("Unable to load resources for " + dbName + "." + fName + ":" + e.getMessage(), e); + return null; + } + + ClassLoader loader = Utilities.getSessionSpecifiedClassLoader(); + Class udfClass = Class.forName(func.getClassName(), true, loader); + // Metastore function should not be native + ret = registerFunction(functionName, udfClass, false); + if (ret == null) + LOG.error(func.getClassName() + " is not a valid UDF class and was not registered."); + } + } catch (HiveException e) { + if (!((e.getCause() != null) && (e.getCause() instanceof MetaException)) && + (e.getCause().getCause() != null) && (e.getCause().getCause() instanceof NoSuchObjectException)) { + LOG.info("Unable to lookup UDF in metastore: " + e); + } + } catch (ClassNotFoundException e) { + // Lookup of UDf class failed + LOG.error("Unable to load UDF class: " + e); + } + + return ret; + } + + private void loadFunctionResourcesIfNecessary(String functionName, FunctionInfo cfi) { + try { + // Check if the necessary JARs have been loaded for this function. + checkFunctionClass(cfi); + } catch (Exception e) { + // Unable to resolve the UDF with the classloader. + // Look up the function in the metastore and load any resources. + LOG.debug("Attempting to reload resources for " + functionName); + try { + String[] parts = FunctionUtils.getQualifiedFunctionNameParts(functionName); + Function func = Hive.get().getFunction(parts[0], parts[1]); + if (func != null) { + FunctionTask.addFunctionResources(func.getResourceUris()); + // Check again now that we've loaded the resources in this thread. + checkFunctionClass(cfi); + } else { + // Couldn't find the function .. just rethrow the original error + LOG.error("Unable to reload resources for " + functionName); + throw e; + } + } catch (Exception err) { + throw new RuntimeException(err); + } + } + } + + private void checkFunctionClass(FunctionInfo cfi) throws ClassNotFoundException { + // This call will fail for non-generic UDFs using GenericUDFBridge + Class udfClass = cfi.getFunctionClass(); + // Even if we have a reference to the class (which will be the case for GenericUDFs), + // the classloader may not be able to resolve the class, which would mean reflection-based + // methods would fail such as for plan deserialization. Make sure this works too. + Class.forName(udfClass.getName(), true, Utilities.getSessionSpecifiedClassLoader()); + } + + public synchronized void clear() { + if (isNative) { + throw new IllegalStateException("System function registry cannot be cleared"); + } + mFunctions.clear(); + nativeUdfs.clear(); + } + + private boolean canLookupMetaStoreFunctions() { + // System registry allowed to lookup metastore for functions + return isNative; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java index efecb05..da92326 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/WindowFunctionInfo.java @@ -22,23 +22,19 @@ import org.apache.hive.common.util.AnnotationUtils; @SuppressWarnings("deprecation") -public class WindowFunctionInfo implements CommonFunctionInfo { - boolean supportsWindow = true; - boolean pivotResult = false; - boolean impliesOrder = false; - FunctionInfo fInfo; - - WindowFunctionInfo(FunctionInfo fInfo) { - assert fInfo.isGenericUDAF(); - this.fInfo = fInfo; - Class wfnCls = fInfo.getGenericUDAFResolver().getClass(); +public class WindowFunctionInfo extends FunctionInfo { + + private final boolean supportsWindow; + private final boolean pivotResult; + private final boolean impliesOrder; + + public WindowFunctionInfo(boolean isNative, String functionName, GenericUDAFResolver resolver) { + super(isNative, functionName, resolver); WindowFunctionDescription def = - AnnotationUtils.getAnnotation(wfnCls, WindowFunctionDescription.class); - if ( def != null) { - supportsWindow = def.supportsWindow(); - pivotResult = def.pivotResult(); - impliesOrder = def.impliesOrder(); - } + AnnotationUtils.getAnnotation(resolver.getClass(), WindowFunctionDescription.class); + supportsWindow = def == null ? true : def.supportsWindow(); + pivotResult = def == null ? false : def.pivotResult(); + impliesOrder = def == null ? false : def.impliesOrder(); } public boolean isSupportsWindow() { @@ -52,12 +48,4 @@ public boolean isPivotResult() { public boolean isImpliesOrder() { return impliesOrder; } - public FunctionInfo getfInfo() { - return fInfo; - } - - @Override - public Class getFunctionClass() { - return getfInfo().getFunctionClass(); - } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java index 22e5b47..37b536d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.ResourceType; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.exec.FunctionInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.FunctionUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -95,13 +96,16 @@ private void analyzeDropFunction(ASTNode ast) throws SemanticException { boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); - if (FunctionRegistry.getFunctionInfo(functionName) == null) { + FunctionInfo info = FunctionRegistry.getFunctionInfo(functionName); + if (info == null) { if (throwException) { throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg(functionName)); } else { // Fail silently return; } + } else if (info.isNative()) { + throw new SemanticException(ErrorMsg.DROP_NATIVE_FUNCTION.getMsg(functionName)); } boolean isTemporaryFunction = (ast.getFirstChildWithType(HiveParser.TOK_TEMPORARY) != null); diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index af633cb..c10c530 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.ql.MapRedStats; +import org.apache.hadoop.hive.ql.exec.Registry; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; @@ -240,6 +241,8 @@ */ private final Set preReloadableAuxJars = new HashSet(); + private final Registry registry = new Registry(); + /** * Get the lineage state stored in this session. * @@ -721,8 +724,21 @@ public static SessionState get() { return tss.get(); } + public static Registry getRegistry() { + SessionState session = get(); + return session != null ? session.registry : null; + } + + public static Registry getRegistryForWrite() { + Registry registry = getRegistry(); + if (registry == null) { + throw new RuntimeException("Function registery for session is not initialized"); + } + return registry; + } + /** - * get hiveHitsory object which does structured logging. + * get hiveHistory object which does structured logging. * * @return The hive history object */ @@ -1228,6 +1244,7 @@ public void setCurrentDatabase(String currentDatabase) { } public void close() throws IOException { + registry.clear();; if (txnMgr != null) txnMgr.closeTxnManager(); JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader); File resourceDir = diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java index 46f8052..57c775c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java @@ -66,12 +66,12 @@ public void testDropMacroDoesNotExist() throws Exception { @Test public void testDropMacroExistsDoNotIgnoreErrors() throws Exception { conf.setBoolVar(ConfVars.DROPIGNORESNONEXISTENT, false); - FunctionRegistry.registerGenericUDF(false, "SOME_MACRO", GenericUDFMacro.class); + FunctionRegistry.registerTemporaryFunction("SOME_MACRO", GenericUDFMacro.class); analyze(parse("DROP TEMPORARY MACRO SOME_MACRO")); } @Test public void testDropMacro() throws Exception { - FunctionRegistry.registerGenericUDF(false, "SOME_MACRO", GenericUDFMacro.class); + FunctionRegistry.registerTemporaryFunction("SOME_MACRO", GenericUDFMacro.class); analyze(parse("DROP TEMPORARY MACRO SOME_MACRO")); } @Test(expected = SemanticException.class) diff --git ql/src/test/queries/clientnegative/drop_native_udf.q ql/src/test/queries/clientnegative/drop_native_udf.q index ae047bb..4188cf2 100644 --- ql/src/test/queries/clientnegative/drop_native_udf.q +++ ql/src/test/queries/clientnegative/drop_native_udf.q @@ -1 +1 @@ -DROP TEMPORARY FUNCTION max; +DROP FUNCTION max; diff --git ql/src/test/results/clientnegative/drop_native_udf.q.out ql/src/test/results/clientnegative/drop_native_udf.q.out index 9f0eaa5..9e8b9ee 100644 --- ql/src/test/results/clientnegative/drop_native_udf.q.out +++ ql/src/test/results/clientnegative/drop_native_udf.q.out @@ -1,4 +1 @@ -PREHOOK: query: DROP TEMPORARY FUNCTION max -PREHOOK: type: DROPFUNCTION -PREHOOK: Output: max -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.FunctionTask +FAILED: SemanticException [Error 10297]: Cannot drop native function max diff --git ql/src/test/results/clientpositive/show_functions.q.out ql/src/test/results/clientpositive/show_functions.q.out index 944c567..9a56745 100644 --- ql/src/test/results/clientpositive/show_functions.q.out +++ ql/src/test/results/clientpositive/show_functions.q.out @@ -18,6 +18,40 @@ POSTHOOK: type: SHOWFUNCTIONS == > >= +@_avg +@_collect_list +@_collect_set +@_compute_stats +@_context_ngrams +@_corr +@_count +@_covar_pop +@_covar_samp +@_cume_dist +@_dense_rank +@_ewah_bitmap +@_first_value +@_histogram_numeric +@_lag +@_last_value +@_lead +@_max +@_min +@_ngrams +@_ntile +@_percent_rank +@_percentile +@_percentile_approx +@_rank +@_row_number +@_std +@_stddev +@_stddev_pop +@_stddev_samp +@_sum +@_var_pop +@_var_samp +@_variance ^ abs acos @@ -227,6 +261,11 @@ PREHOOK: query: SHOW FUNCTIONS '.*e$' PREHOOK: type: SHOWFUNCTIONS POSTHOOK: query: SHOW FUNCTIONS '.*e$' POSTHOOK: type: SHOWFUNCTIONS +@_first_value +@_last_value +@_ntile +@_percentile +@_variance assert_true case coalesce diff --git ql/src/test/results/clientpositive/udaf_collect_set.q.out ql/src/test/results/clientpositive/udaf_collect_set.q.out index 42f2d9d..713c911 100644 --- ql/src/test/results/clientpositive/udaf_collect_set.q.out +++ ql/src/test/results/clientpositive/udaf_collect_set.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_set POSTHOOK: type: DESCFUNCTION collect_set(x) - Returns a set of objects with duplicate elements eliminated +Synonyms: @_collect_set PREHOOK: query: DESCRIBE FUNCTION collect_list PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION collect_list @@ -18,6 +19,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED collect_list POSTHOOK: type: DESCFUNCTION collect_list(x) - Returns a list of objects with duplicates +Synonyms: @_collect_list PREHOOK: query: SELECT key, collect_set(value) FROM src GROUP BY key ORDER BY key limit 20 diff --git ql/src/test/results/clientpositive/udaf_corr.q.out ql/src/test/results/clientpositive/udaf_corr.q.out index 1b468c4..d9333ca 100644 --- ql/src/test/results/clientpositive/udaf_corr.q.out +++ ql/src/test/results/clientpositive/udaf_corr.q.out @@ -36,6 +36,7 @@ POSTHOOK: query: DESCRIBE FUNCTION EXTENDED corr POSTHOOK: type: DESCFUNCTION corr(x,y) - Returns the Pearson coefficient of correlation between a set of number pairs +Synonyms: @_corr The function takes as arguments any pair of numeric types and returns a double. Any pair with a NULL is ignored. If the function is applied to an empty set or a singleton set, NULL will be returned. Otherwise, it computes the following: diff --git ql/src/test/results/clientpositive/udaf_covar_pop.q.out ql/src/test/results/clientpositive/udaf_covar_pop.q.out index fa1d47e..2a5337e 100644 --- ql/src/test/results/clientpositive/udaf_covar_pop.q.out +++ ql/src/test/results/clientpositive/udaf_covar_pop.q.out @@ -34,6 +34,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED covar_pop POSTHOOK: type: DESCFUNCTION covar_pop(x,y) - Returns the population covariance of a set of number pairs +Synonyms: @_covar_pop The function takes as arguments any pair of numeric types and returns a double. Any pair with a NULL is ignored. If the function is applied to an empty set, NULL will be returned. Otherwise, it computes the following: diff --git ql/src/test/results/clientpositive/udaf_covar_samp.q.out ql/src/test/results/clientpositive/udaf_covar_samp.q.out index ff5b47e..b0d1ee7 100644 --- ql/src/test/results/clientpositive/udaf_covar_samp.q.out +++ ql/src/test/results/clientpositive/udaf_covar_samp.q.out @@ -34,6 +34,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED covar_samp POSTHOOK: type: DESCFUNCTION covar_samp(x,y) - Returns the sample covariance of a set of number pairs +Synonyms: @_covar_samp The function takes as arguments any pair of numeric types and returns a double. Any pair with a NULL is ignored. If the function is applied to an empty set, NULL will be returned. Otherwise, it computes the following: diff --git ql/src/test/results/clientpositive/udf_avg.q.out ql/src/test/results/clientpositive/udf_avg.q.out index 7ac139f..4759a19 100644 --- ql/src/test/results/clientpositive/udf_avg.q.out +++ ql/src/test/results/clientpositive/udf_avg.q.out @@ -8,3 +8,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED avg POSTHOOK: type: DESCFUNCTION avg(x) - Returns the mean of a set of numbers +Synonyms: @_avg diff --git ql/src/test/results/clientpositive/udf_count.q.out ql/src/test/results/clientpositive/udf_count.q.out index 78228ec..409d1db 100644 --- ql/src/test/results/clientpositive/udf_count.q.out +++ ql/src/test/results/clientpositive/udf_count.q.out @@ -12,6 +12,7 @@ POSTHOOK: type: DESCFUNCTION count(*) - Returns the total number of retrieved rows, including rows containing NULL values. count(expr) - Returns the number of rows for which the supplied expression is non-NULL. count(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-NULL. +Synonyms: @_count PREHOOK: query: EXPLAIN SELECT count(key) FROM src PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN SELECT count(key) FROM src diff --git ql/src/test/results/clientpositive/udf_max.q.out ql/src/test/results/clientpositive/udf_max.q.out index 8535752..a5ef65b 100644 --- ql/src/test/results/clientpositive/udf_max.q.out +++ ql/src/test/results/clientpositive/udf_max.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED max POSTHOOK: type: DESCFUNCTION max(expr) - Returns the maximum value of expr +Synonyms: @_max PREHOOK: query: SELECT max(struct(CAST(key as INT), value)), max(struct(key, value)) FROM src diff --git ql/src/test/results/clientpositive/udf_min.q.out ql/src/test/results/clientpositive/udf_min.q.out index ca8158e..d824746 100644 --- ql/src/test/results/clientpositive/udf_min.q.out +++ ql/src/test/results/clientpositive/udf_min.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED min POSTHOOK: type: DESCFUNCTION min(expr) - Returns the minimum value of expr +Synonyms: @_min PREHOOK: query: SELECT min(struct(CAST(key as INT), value)), min(struct(key, value)) FROM src diff --git ql/src/test/results/clientpositive/udf_percentile.q.out ql/src/test/results/clientpositive/udf_percentile.q.out index b963236..672b29b 100644 --- ql/src/test/results/clientpositive/udf_percentile.q.out +++ ql/src/test/results/clientpositive/udf_percentile.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED percentile POSTHOOK: type: DESCFUNCTION percentile(expr, pc) - Returns the percentile(s) of expr at pc (range: [0,1]).pc can be a double or double array +Synonyms: @_percentile PREHOOK: query: SELECT CAST(key AS INT) DIV 10, percentile(CAST(substr(value, 5) AS INT), 0.0), percentile(CAST(substr(value, 5) AS INT), 0.5), diff --git ql/src/test/results/clientpositive/udf_std.q.out ql/src/test/results/clientpositive/udf_std.q.out index c004157..bfc31a8 100644 --- ql/src/test/results/clientpositive/udf_std.q.out +++ ql/src/test/results/clientpositive/udf_std.q.out @@ -8,4 +8,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED std POSTHOOK: type: DESCFUNCTION std(x) - Returns the standard deviation of a set of numbers -Synonyms: stddev_pop, stddev +Synonyms: @_stddev, stddev_pop, @_std, stddev, @_stddev_pop diff --git ql/src/test/results/clientpositive/udf_stddev.q.out ql/src/test/results/clientpositive/udf_stddev.q.out index 8948dfc..191600e 100644 --- ql/src/test/results/clientpositive/udf_stddev.q.out +++ ql/src/test/results/clientpositive/udf_stddev.q.out @@ -8,4 +8,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED stddev POSTHOOK: type: DESCFUNCTION stddev(x) - Returns the standard deviation of a set of numbers -Synonyms: stddev_pop, std +Synonyms: @_stddev, stddev_pop, @_std, std, @_stddev_pop diff --git ql/src/test/results/clientpositive/udf_stddev_samp.q.out ql/src/test/results/clientpositive/udf_stddev_samp.q.out index 6619ea1..26bac6f 100644 --- ql/src/test/results/clientpositive/udf_stddev_samp.q.out +++ ql/src/test/results/clientpositive/udf_stddev_samp.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp POSTHOOK: type: DESCFUNCTION stddev_samp(x) - Returns the sample standard deviation of a set of numbers +Synonyms: @_stddev_samp PREHOOK: query: DESCRIBE FUNCTION stddev_samp PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION stddev_samp @@ -18,3 +19,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED stddev_samp POSTHOOK: type: DESCFUNCTION stddev_samp(x) - Returns the sample standard deviation of a set of numbers +Synonyms: @_stddev_samp diff --git ql/src/test/results/clientpositive/udf_sum.q.out ql/src/test/results/clientpositive/udf_sum.q.out index 2f4e3da..f763a45 100644 --- ql/src/test/results/clientpositive/udf_sum.q.out +++ ql/src/test/results/clientpositive/udf_sum.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED sum POSTHOOK: type: DESCFUNCTION sum(x) - Returns the sum of a set of numbers +Synonyms: @_sum PREHOOK: query: DESCRIBE FUNCTION sum PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION sum @@ -18,3 +19,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED sum POSTHOOK: type: DESCFUNCTION sum(x) - Returns the sum of a set of numbers +Synonyms: @_sum diff --git ql/src/test/results/clientpositive/udf_var_pop.q.out ql/src/test/results/clientpositive/udf_var_pop.q.out index cd9c61e..476f504 100644 --- ql/src/test/results/clientpositive/udf_var_pop.q.out +++ ql/src/test/results/clientpositive/udf_var_pop.q.out @@ -8,4 +8,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_pop POSTHOOK: type: DESCFUNCTION var_pop(x) - Returns the variance of a set of numbers -Synonyms: variance +Synonyms: variance, @_var_pop, @_variance diff --git ql/src/test/results/clientpositive/udf_var_samp.q.out ql/src/test/results/clientpositive/udf_var_samp.q.out index 190435d..5dc0ad9 100644 --- ql/src/test/results/clientpositive/udf_var_samp.q.out +++ ql/src/test/results/clientpositive/udf_var_samp.q.out @@ -8,6 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp POSTHOOK: type: DESCFUNCTION var_samp(x) - Returns the sample variance of a set of numbers +Synonyms: @_var_samp PREHOOK: query: DESCRIBE FUNCTION var_samp PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION var_samp @@ -18,3 +19,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_samp POSTHOOK: type: DESCFUNCTION var_samp(x) - Returns the sample variance of a set of numbers +Synonyms: @_var_samp diff --git ql/src/test/results/clientpositive/udf_variance.q.out ql/src/test/results/clientpositive/udf_variance.q.out index c00dd66..029f72a 100644 --- ql/src/test/results/clientpositive/udf_variance.q.out +++ ql/src/test/results/clientpositive/udf_variance.q.out @@ -8,7 +8,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED variance POSTHOOK: type: DESCFUNCTION variance(x) - Returns the variance of a set of numbers -Synonyms: var_pop +Synonyms: @_var_pop, var_pop, @_variance PREHOOK: query: DESCRIBE FUNCTION var_pop PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION var_pop @@ -19,7 +19,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_pop POSTHOOK: type: DESCFUNCTION var_pop(x) - Returns the variance of a set of numbers -Synonyms: variance +Synonyms: variance, @_var_pop, @_variance PREHOOK: query: DESCRIBE FUNCTION variance PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION variance @@ -30,7 +30,7 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED variance POSTHOOK: type: DESCFUNCTION variance(x) - Returns the variance of a set of numbers -Synonyms: var_pop +Synonyms: @_var_pop, var_pop, @_variance PREHOOK: query: DESCRIBE FUNCTION var_pop PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION var_pop @@ -41,4 +41,4 @@ PREHOOK: type: DESCFUNCTION POSTHOOK: query: DESCRIBE FUNCTION EXTENDED var_pop POSTHOOK: type: DESCFUNCTION var_pop(x) - Returns the variance of a set of numbers -Synonyms: variance +Synonyms: variance, @_var_pop, @_variance diff --git service/src/test/org/apache/hadoop/hive/service/TestHiveServerSessions.java service/src/test/org/apache/hadoop/hive/service/TestHiveServerSessions.java index fd38907..c33cada 100644 --- service/src/test/org/apache/hadoop/hive/service/TestHiveServerSessions.java +++ service/src/test/org/apache/hadoop/hive/service/TestHiveServerSessions.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.service; import junit.framework.TestCase; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.transport.TSocket; @@ -38,8 +40,16 @@ private TSocket[] transports = new TSocket[clientNum]; private HiveClient[] clients = new HiveClient[clientNum]; + private String tableName; + private Path dataFilePath; + public TestHiveServerSessions(String name) { super(name); + HiveConf conf = new HiveConf(TestHiveServerSessions.class); + String dataFileDir = conf.get("test.data.files").replace('\\', '/') + .replace("c:", ""); + dataFilePath = new Path(dataFileDir, "kv1.txt"); + tableName = "testsessionfunc"; } @Override @@ -59,6 +69,7 @@ public void run() { transport.open(); transports[i] = transport; clients[i] = new HiveClient(new TBinaryProtocol(transport)); + clients[i].execute("set hive.support.concurrency = false"); } } @@ -96,4 +107,26 @@ public void testSessionVars() throws Exception { assertEquals("hiveconf:var=value" + i, clients[i].fetchOne()); } } + + public void testSessionFuncs() throws Exception { + + try { + clients[0].execute("drop table " + tableName); + } catch (Exception ex) { + } + clients[0].execute("create table " + tableName + " (key int, value string)"); + clients[0].execute("load data local inpath '" + dataFilePath.toString() + + "' into table " + tableName); + + clients[0].execute("create temporary function dummy as " + + "'org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan'"); + clients[1].execute("create temporary function dummy as " + + "'org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan'"); + + clients[0].execute("select count(*) from " + tableName + " where dummy(key, 100)"); + assertEquals(84, Integer.valueOf(clients[0].fetchOne()).intValue()); + + clients[1].execute("select count(*) from " + tableName + " where dummy(key, 100)"); + assertEquals(416, Integer.valueOf(clients[1].fetchOne()).intValue()); + } }