[FLINK-28451] HiveFunctionDefinitionFactory load udf using user classloader instead of thread context classloader

• Type: Sub-task
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 1.16.0
• Fix Version/s: 1.16.0
• Component/s: Connectors / Hive
• Labels: pull-request-available
• Assignee: luoyuxia
• Reporter: dalongliu
• Created: 07/Jul/22 17:45
• Updated: 23/Jul/22 04:44
• Resolved: 23/Jul/22 04:44

Description

Currently `HiveFunctionDefinitionFactory` load user defined function using thread context classloader, after we introducing user classloader to load function class from FLINK-27659, this maybe doesn't work if the user jar not in thread classloader. The following exception will be thrown:

 23478 [main] WARN  org.apache.flink.table.client.cli.CliClient [] - Could not execute SQL statement.org.apache.flink.table.client.gateway.SqlExecutionException: Failed to parse statement: SELECT id, func1(str) FROM (VALUES (1, 'Hello World')) AS T(id, str) ;    at org.apache.flink.table.client.gateway.local.LocalExecutor.parseStatement(LocalExecutor.java:174) ~[classes/:?]    at org.apache.flink.table.client.cli.SqlCommandParserImpl.parseCommand(SqlCommandParserImpl.java:45) ~[classes/:?]    at org.apache.flink.table.client.cli.SqlMultiLineParser.parse(SqlMultiLineParser.java:71) ~[classes/:?]    at org.jline.reader.impl.LineReaderImpl.acceptLine(LineReaderImpl.java:2964) ~[jline-reader-3.21.0.jar:?]    at org.jline.reader.impl.LineReaderImpl$1.apply(LineReaderImpl.java:3778) ~[jline-reader-3.21.0.jar:?]    at org.jline.reader.impl.LineReaderImpl.readLine(LineReaderImpl.java:679) ~[jline-reader-3.21.0.jar:?]    at org.apache.flink.table.client.cli.CliClient.getAndExecuteStatements(CliClient.java:296) [classes/:?]    at org.apache.flink.table.client.cli.CliClient.executeInteractive(CliClient.java:281) [classes/:?]    at org.apache.flink.table.client.cli.CliClient.executeInInteractiveMode(CliClient.java:229) [classes/:?]    at org.apache.flink.table.client.cli.CliClientITCase.runSqlStatements(CliClientITCase.java:172) [test-classes/:?]    at org.apache.flink.table.client.cli.CliClientITCase.testSqlStatements(CliClientITCase.java:137) [test-classes/:?]    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_291]    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_291]    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_291]    at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_291]    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) [junit-4.13.2.jar:4.13.2]    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) [junit-4.13.2.jar:4.13.2]    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) [junit-4.13.2.jar:4.13.2]    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) [junit-4.13.2.jar:4.13.2]    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) [junit-4.13.2.jar:4.13.2]    at org.apache.flink.util.TestNameProvider$1.evaluate(TestNameProvider.java:45) [classes/:?]    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) [junit-4.13.2.jar:4.13.2]    at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) [junit-4.13.2.jar:4.13.2]    at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) [junit-4.13.2.jar:4.13.2]    at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54) [junit-4.13.2.jar:4.13.2]    at org.junit.rules.RunRules.evaluate(RunRules.java:20) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]    at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]    at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]    at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69) [junit-rt.jar:?]    at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38) [junit-rt.jar:?]    at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11) [idea_rt.jar:?]    at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35) [junit-rt.jar:?]    at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:235) [junit-rt.jar:?]    at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:54) [junit-rt.jar:?]Caused by: org.apache.flink.table.api.ValidationException: SQL validation failed. Can't resolve udf class LowerUDF    at org.apache.flink.table.planner.calcite.FlinkPlannerImpl.org$apache$flink$table$planner$calcite$FlinkPlannerImpl$$validate(FlinkPlannerImpl.scala:182) ~[classes/:?]    at org.apache.flink.table.planner.calcite.FlinkPlannerImpl.validate(FlinkPlannerImpl.scala:109) ~[classes/:?]    at org.apache.flink.table.planner.operations.SqlToOperationConverter.convert(SqlToOperationConverter.java:241) ~[classes/:?]    at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:106) ~[classes/:?]    at org.apache.flink.table.client.gateway.local.LocalExecutor.parseStatement(LocalExecutor.java:172) ~[classes/:?]    ... 54 moreCaused by: java.lang.RuntimeException: Can't resolve udf class LowerUDF    at org.apache.flink.table.catalog.CatalogFunctionImpl.isGeneric(CatalogFunctionImpl.java:90) ~[classes/:?]    at org.apache.flink.table.catalog.hive.factories.HiveFunctionDefinitionFactory.createFunctionDefinition(HiveFunctionDefinitionFactory.java:61) ~[classes/:?]    at org.apache.flink.table.factories.FunctionDefinitionFactory.createFunctionDefinition(FunctionDefinitionFactory.java:60) ~[classes/:?]    at org.apache.flink.table.catalog.FunctionCatalog.resolvePreciseFunctionReference(FunctionCatalog.java:586) ~[classes/:?]    at org.apache.flink.table.catalog.FunctionCatalog.lambda$resolveAmbiguousFunctionReference$3(FunctionCatalog.java:634) ~[classes/:?]    at java.util.Optional.orElseGet(Optional.java:267) ~[?:1.8.0_291]    at org.apache.flink.table.catalog.FunctionCatalog.resolveAmbiguousFunctionReference(FunctionCatalog.java:634) ~[classes/:?]    at org.apache.flink.table.catalog.FunctionCatalog.lookupFunction(FunctionCatalog.java:368) ~[classes/:?]    at org.apache.flink.table.planner.catalog.FunctionCatalogOperatorTable.lookupOperatorOverloads(FunctionCatalogOperatorTable.java:94) ~[classes/:?]    at org.apache.calcite.sql.util.ChainedSqlOperatorTable.lookupOperatorOverloads(ChainedSqlOperatorTable.java:67) ~[calcite-core-1.26.0.jar:1.26.0]    at org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1183) ~[classes/:1.26.0]    at org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1200) ~[classes/:1.26.0]    at org.apache.calcite.sql.validate.SqlValidatorImpl.performUnconditionalRewrites(SqlValidatorImpl.java:1169) ~[classes/:1.26.0]    at org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:945) ~[classes/:1.26.0]    at org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:704) ~[classes/:1.26.0]    at org.apache.flink.table.planner.calcite.FlinkPlannerImpl.org$apache$flink$table$planner$calcite$FlinkPlannerImpl$$validate(FlinkPlannerImpl.scala:178) ~[classes/:?]    at org.apache.flink.table.planner.calcite.FlinkPlannerImpl.validate(FlinkPlannerImpl.scala:109) ~[classes/:?]    at org.apache.flink.table.planner.operations.SqlToOperationConverter.convert(SqlToOperationConverter.java:241) ~[classes/:?]    at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:106) ~[classes/:?]    at org.apache.flink.table.client.gateway.local.LocalExecutor.parseStatement(LocalExecutor.java:172) ~[classes/:?]    ... 54 more 

Issue Links

• duplicates: FLINK-28430 HiveFunctionWrapper should use UserClassLoader to load class
• links to: GitHub Pull Request #20211