diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 96a78fc..e03614b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.udf.SettableUDF; import org.apache.hadoop.hive.ql.udf.UDAFPercentile; import org.apache.hadoop.hive.ql.udf.UDFAcos; @@ -167,7 +168,6 @@ static Map windowFunctions = Collections.synchronizedMap(new LinkedHashMap()); - static { registerGenericUDF("concat", GenericUDFConcat.class); registerUDF("substr", UDFSubstr.class, false); @@ -417,8 +417,8 @@ registerGenericUDTF("stack", GenericUDTFStack.class); //PTF declarations - registerGenericUDF(true, LEAD_FUNC_NAME, GenericUDFLead.class); - registerGenericUDF(true, LAG_FUNC_NAME, GenericUDFLag.class); + registerGenericUDF(LEAD_FUNC_NAME, GenericUDFLead.class); + registerGenericUDF(LAG_FUNC_NAME, GenericUDFLag.class); registerHiveUDAFsAsWindowFunctions(); registerWindowFunction("row_number", new GenericUDAFRowNumber()); @@ -518,8 +518,32 @@ public static void registerGenericUDTF(boolean isNative, String functionName, } } + private static T getQualifiedFunctionInfo(Map mFunctions, String functionName) { + T functionInfo = mFunctions.get(functionName); + // Eventually this would check metastore for registered functions. + return functionInfo; + } + + private static T getFunctionInfo(Map mFunctions, String functionName) { + functionName = functionName.toLowerCase(); + T functionInfo = null; + if (FunctionUtils.isQualifiedFunctionName(functionName)) { + functionInfo = getQualifiedFunctionInfo(mFunctions, functionName); + } else { + // First try without qualifiers - would resolve builtin/temp functions. + // Otherwise try qualifying with current db name. + functionInfo = mFunctions.get(functionName); + if (functionInfo == null && !FunctionUtils.isQualifiedFunctionName(functionName)) { + String qualifiedName = FunctionUtils.qualifyFunctionName(functionName, + SessionState.get().getCurrentDatabase()); + functionInfo = getQualifiedFunctionInfo(mFunctions, qualifiedName); + } + } + return functionInfo; + } + public static FunctionInfo getFunctionInfo(String functionName) { - return mFunctions.get(functionName.toLowerCase()); + return getFunctionInfo(mFunctions, functionName); } /** @@ -1040,7 +1064,7 @@ public static GenericUDAFResolver getGenericUDAFResolver(String functionName) { if (LOG.isDebugEnabled()) { LOG.debug("Looking up GenericUDAF: " + functionName); } - FunctionInfo finfo = mFunctions.get(functionName.toLowerCase()); + FunctionInfo finfo = getFunctionInfo(functionName); if (finfo == null) { return null; } @@ -1688,9 +1712,8 @@ public static void registerWindowFunction(String name, GenericUDAFResolver wFn, windowFunctions.put(name.toLowerCase(), wInfo); } - public static WindowFunctionInfo getWindowFunctionInfo(String name) - { - return windowFunctions.get(name.toLowerCase()); + public static WindowFunctionInfo getWindowFunctionInfo(String functionName) { + return getFunctionInfo(windowFunctions, functionName); } /** @@ -1703,7 +1726,7 @@ public static WindowFunctionInfo getWindowFunctionInfo(String name) */ public static boolean impliesOrder(String functionName) { - FunctionInfo info = mFunctions.get(functionName.toLowerCase()); + FunctionInfo info = getFunctionInfo(functionName); if (info != null) { if (info.isGenericUDF()) { UDFType type = info.getGenericUDF().getClass().getAnnotation(UDFType.class); @@ -1712,7 +1735,7 @@ public static boolean impliesOrder(String functionName) { } } } - WindowFunctionInfo windowInfo = windowFunctions.get(functionName.toLowerCase()); + WindowFunctionInfo windowInfo = getWindowFunctionInfo(functionName); if (windowInfo != null) { return windowInfo.isImpliesOrder(); } @@ -1735,13 +1758,13 @@ static void registerHiveUDAFsAsWindowFunctions() public static boolean isTableFunction(String name) { - FunctionInfo tFInfo = mFunctions.get(name.toLowerCase()); + FunctionInfo tFInfo = getFunctionInfo(name); return tFInfo != null && !tFInfo.isInternalTableFunction() && tFInfo.isTableFunction(); } public static TableFunctionResolver getTableFunctionResolver(String name) { - FunctionInfo tfInfo = mFunctions.get(name.toLowerCase()); + FunctionInfo tfInfo = getFunctionInfo(name); if(tfInfo.isTableFunction()) { return (TableFunctionResolver) ReflectionUtils.newInstance(tfInfo.getFunctionClass(), null); } @@ -1774,7 +1797,7 @@ public static void registerTableFunction(String name, Class= 0; + } + + public static String qualifyFunctionName(String functionName, String dbName) { + if (isQualifiedFunctionName(functionName)) { + return functionName; + } + return dbName + "." + functionName; + } + + /** + * Splits a qualified function name into an array containing the database name and function name. + * If the name is not qualified, the database name is null. + * If there is more than one '.', an exception will be thrown. + * @param functionName Function name, which may or may not be qualified + * @return + */ + public static String[] splitQualifiedFunctionName(String functionName) throws HiveException { + String[] names = functionName.split("\\."); + if (names.length == 1) { + String[] retval = { null, functionName }; + return retval; + } else if (names.length > 2) { + throw new HiveException("Function name does not have correct format: " + functionName); + } + return names; + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java index da917f7..cc12f30 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.FunctionUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc; import org.apache.hadoop.hive.ql.plan.DropFunctionDesc; @@ -55,6 +56,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { private void analyzeCreateFunction(ASTNode ast) throws SemanticException { String functionName = ast.getChild(0).getText(); String className = unescapeSQLString(ast.getChild(1).getText()); + + // Temp functions are not allowed to have qualified names. + if (FunctionUtils.isQualifiedFunctionName(functionName)) { + throw new SemanticException("Temporary function cannot be created with a qualified name."); + } + CreateFunctionDesc desc = new CreateFunctionDesc(functionName, className); rootTasks.add(TaskFactory.get(new FunctionWork(desc), conf)); } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index cd52e47..3537e2b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -1242,7 +1242,7 @@ showStatement | KW_SHOW KW_TABLES ((KW_FROM|KW_IN) db_name=identifier)? (KW_LIKE showStmtIdentifier|showStmtIdentifier)? -> ^(TOK_SHOWTABLES (TOK_FROM $db_name)? showStmtIdentifier?) | KW_SHOW KW_COLUMNS (KW_FROM|KW_IN) tabname=tableName ((KW_FROM|KW_IN) db_name=identifier)? -> ^(TOK_SHOWCOLUMNS $db_name? $tabname) - | KW_SHOW KW_FUNCTIONS showStmtIdentifier? -> ^(TOK_SHOWFUNCTIONS showStmtIdentifier?) + | KW_SHOW KW_FUNCTIONS functionIdentifier? -> ^(TOK_SHOWFUNCTIONS functionIdentifier?) | KW_SHOW KW_PARTITIONS tabName=tableName partitionSpec? -> ^(TOK_SHOWPARTITIONS $tabName partitionSpec?) | KW_SHOW KW_CREATE KW_TABLE tabName=tableName -> ^(TOK_SHOW_CREATETABLE $tabName) | KW_SHOW KW_TABLE KW_EXTENDED ((KW_FROM|KW_IN) db_name=identifier)? KW_LIKE showStmtIdentifier partitionSpec? @@ -1440,30 +1440,30 @@ metastoreCheck createFunctionStatement @init { msgs.push("create function statement"); } @after { msgs.pop(); } - : KW_CREATE KW_TEMPORARY KW_FUNCTION identifier KW_AS StringLiteral - -> ^(TOK_CREATEFUNCTION identifier StringLiteral) + : KW_CREATE KW_TEMPORARY KW_FUNCTION functionIdentifier KW_AS StringLiteral + -> ^(TOK_CREATEFUNCTION functionIdentifier StringLiteral) ; dropFunctionStatement @init { msgs.push("drop temporary function statement"); } @after { msgs.pop(); } - : KW_DROP KW_TEMPORARY KW_FUNCTION ifExists? identifier - -> ^(TOK_DROPFUNCTION identifier ifExists?) + : KW_DROP KW_TEMPORARY KW_FUNCTION ifExists? functionIdentifier + -> ^(TOK_DROPFUNCTION functionIdentifier ifExists?) ; createMacroStatement @init { msgs.push("create macro statement"); } @after { msgs.pop(); } - : KW_CREATE KW_TEMPORARY KW_MACRO Identifier + : KW_CREATE KW_TEMPORARY KW_MACRO functionIdentifier LPAREN columnNameTypeList? RPAREN expression - -> ^(TOK_CREATEMACRO Identifier columnNameTypeList? expression) + -> ^(TOK_CREATEMACRO functionIdentifier columnNameTypeList? expression) ; dropMacroStatement @init { msgs.push("drop macro statement"); } @after { msgs.pop(); } - : KW_DROP KW_TEMPORARY KW_MACRO ifExists? Identifier - -> ^(TOK_DROPMACRO Identifier ifExists?) + : KW_DROP KW_TEMPORARY KW_MACRO ifExists? functionIdentifier + -> ^(TOK_DROPMACRO functionIdentifier ifExists?) ; createViewStatement diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index 9f279d2..b66d379 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -183,7 +183,7 @@ functionName @init { gParent.msgs.push("function name"); } @after { gParent.msgs.pop(); } : // Keyword IF is also a function name - KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | identifier + KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | functionIdentifier ; castExpression @@ -268,7 +268,7 @@ atomExpression KW_NULL -> TOK_NULL | dateLiteral | constant - | function + | (functionName LPAREN) => function | castExpression | caseExpression | whenExpression @@ -524,7 +524,7 @@ descFuncNames : sysFuncNames | StringLiteral - | identifier + | functionIdentifier ; identifier @@ -533,6 +533,15 @@ identifier | nonReserved -> Identifier[$nonReserved.text] ; +functionIdentifier +@init { gParent.msgs.push("function identifier"); } +@after { gParent.msgs.pop(); } + : db=identifier DOT fn=identifier + -> Identifier[$db.text + "." + $fn.text] + | + identifier + ; + nonReserved : KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION | KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_ROLES | KW_INNER | KW_DEFINED | KW_ADMIN diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java index b42a425..36f8f71 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/MacroSemanticAnalyzer.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; +import org.apache.hadoop.hive.ql.exec.FunctionUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.Node; @@ -73,6 +74,12 @@ public void analyzeInternal(ASTNode ast) throws SemanticException { @SuppressWarnings("unchecked") private void analyzeCreateMacro(ASTNode ast) throws SemanticException { String functionName = ast.getChild(0).getText(); + + // Temp macros are not allowed to have qualified names. + if (FunctionUtils.isQualifiedFunctionName(functionName)) { + throw new SemanticException("Temporary macro cannot be created with a qualified name."); + } + List arguments = BaseSemanticAnalyzer.getColumns((ASTNode)ast.getChild(1), true); boolean isNoArgumentMacro = arguments.size() == 0; @@ -80,6 +87,7 @@ private void analyzeCreateMacro(ASTNode ast) throws SemanticException { ArrayList macroColNames = new ArrayList(arguments.size()); ArrayList macroColTypes = new ArrayList(arguments.size()); final Set actualColumnNames = new HashSet(); + if(!isNoArgumentMacro) { /* * Walk down expression to see which arguments are actually used. diff --git ql/src/test/queries/clientnegative/udf_invalid.q ql/src/test/queries/clientnegative/udf_invalid.q new file mode 100644 index 0000000..68050fd --- /dev/null +++ ql/src/test/queries/clientnegative/udf_invalid.q @@ -0,0 +1 @@ +select default.nonexistfunc() from src; diff --git ql/src/test/queries/clientnegative/udf_qualified_name.q ql/src/test/queries/clientnegative/udf_qualified_name.q new file mode 100644 index 0000000..476dfa2 --- /dev/null +++ ql/src/test/queries/clientnegative/udf_qualified_name.q @@ -0,0 +1 @@ +create temporary function default.myfunc as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum'; diff --git ql/src/test/results/clientnegative/udf_invalid.q.out ql/src/test/results/clientnegative/udf_invalid.q.out new file mode 100644 index 0000000..c394cc8 --- /dev/null +++ ql/src/test/results/clientnegative/udf_invalid.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Line 0:-1 Invalid function 'default.nonexistfunc' diff --git ql/src/test/results/clientnegative/udf_qualified_name.q.out ql/src/test/results/clientnegative/udf_qualified_name.q.out new file mode 100644 index 0000000..69404b4 --- /dev/null +++ ql/src/test/results/clientnegative/udf_qualified_name.q.out @@ -0,0 +1 @@ +FAILED: SemanticException Temporary function cannot be created with a qualified name.