Details
-
Improvement
-
Status: Open
-
Major
-
Resolution: Unresolved
-
None
-
None
-
None
Description
CalcitePlanner internally uses org.codehaus.janino.UnitCompiler (calcite dependency) and this appears to load the jars in every thread. Need to check if this can be avoided.
Here is an example.
at java.util.zip.ZipFile.getEntry(Native Method) at java.util.zip.ZipFile.getEntry(ZipFile.java:310) - locked <0x00000005c1af21c0> (a java.util.jar.JarFile) at java.util.jar.JarFile.getEntry(JarFile.java:240) at java.util.jar.JarFile.getJarEntry(JarFile.java:223) at sun.misc.URLClassPath$JarLoader.getResource(URLClassPath.java:1005) at sun.misc.URLClassPath.getResource(URLClassPath.java:212) at java.net.URLClassLoader$1.run(URLClassLoader.java:365) at java.net.URLClassLoader$1.run(URLClassLoader.java:362) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:361) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) - locked <0x00000005caa3be88> (a java.lang.Object) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.codehaus.janino.ClassLoaderIClassLoader.findIClass(ClassLoaderIClassLoader.java:89) at org.codehaus.janino.IClassLoader.loadIClass(IClassLoader.java:312) - locked <0x0000000686136868> (a org.codehaus.janino.ClassLoaderIClassLoader) at org.codehaus.janino.UnitCompiler.findTypeByName(UnitCompiler.java:8556) at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8478) at org.codehaus.janino.UnitCompiler.reclassifyName(UnitCompiler.java:8471) at org.codehaus.janino.UnitCompiler.reclassify(UnitCompiler.java:8331) at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:6855) at org.codehaus.janino.UnitCompiler.access$14200(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6497) at org.codehaus.janino.UnitCompiler$22$2$1.visitAmbiguousName(UnitCompiler.java:6494) at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:4224) at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6494) at org.codehaus.janino.UnitCompiler$22$2.visitLvalue(UnitCompiler.java:6490) at org.codehaus.janino.Java$Lvalue.accept(Java.java:4148) at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490) at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469) at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116) at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469) at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9026) at org.codehaus.janino.UnitCompiler.getType2(UnitCompiler.java:7106) at org.codehaus.janino.UnitCompiler.access$15800(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6517) at org.codehaus.janino.UnitCompiler$22$2.visitMethodInvocation(UnitCompiler.java:6490) at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073) at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6490) at org.codehaus.janino.UnitCompiler$22.visitRvalue(UnitCompiler.java:6469) at org.codehaus.janino.Java$Rvalue.accept(Java.java:4116) at org.codehaus.janino.UnitCompiler.getType(UnitCompiler.java:6469) at org.codehaus.janino.UnitCompiler.findMostSpecificIInvocable(UnitCompiler.java:9237) at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9123) at org.codehaus.janino.UnitCompiler.findIMethod(UnitCompiler.java:9025) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5062) at org.codehaus.janino.UnitCompiler.access$9100(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4423) at org.codehaus.janino.UnitCompiler$16.visitMethodInvocation(UnitCompiler.java:4396) at org.codehaus.janino.Java$MethodInvocation.accept(Java.java:5073) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5622) at org.codehaus.janino.UnitCompiler.compileGet2(UnitCompiler.java:5592) at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4434) at org.codehaus.janino.UnitCompiler$16.visitNewInitializedArray(UnitCompiler.java:4396) at org.codehaus.janino.Java$NewInitializedArray.accept(Java.java:5373) at org.codehaus.janino.UnitCompiler.compileGet(UnitCompiler.java:4396) at org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:5662) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2649) at org.codehaus.janino.UnitCompiler.access$2800(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1504) at org.codehaus.janino.UnitCompiler$6.visitReturnStatement(UnitCompiler.java:1487) at org.codehaus.janino.Java$ReturnStatement.accept(Java.java:3563) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1487) at org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1567) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3388) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1357) at org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1330) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:822) at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:432) at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:215) at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:411) at org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:406) at org.codehaus.janino.Java$PackageMemberClassDeclaration.accept(Java.java:1414) at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:406) at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:378) at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:237) at org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:465) at org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:313) at org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:235) at org.apache.calcite.rex.RexExecutable.compile(RexExecutable.java:60) at org.apache.calcite.rex.RexExecutable.<init>(RexExecutable.java:49) at org.apache.calcite.rex.RexExecutorImpl.reduce(RexExecutorImpl.java:127) at org.apache.calcite.rex.RexSimplify.simplifyCast(RexSimplify.java:1810) at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:270) at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473) at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:321) at org.apache.calcite.rex.RexSimplify.simplifyComparison(RexSimplify.java:313) at org.apache.calcite.rex.RexSimplify.simplify(RexSimplify.java:288) at org.apache.calcite.rex.RexSimplify.simplifyList(RexSimplify.java:473) at org.apache.calcite.rex.RexSimplify.simplifyAnds(RexSimplify.java:463) at org.apache.calcite.rex.RexUtil.simplifyAnds(RexUtil.java:1748) at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:759) at org.apache.hadoop.hive.ql.optimizer.calcite.HiveSubQRemoveRelBuilder.filter(HiveSubQRemoveRelBuilder.java:749) at org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveSubQueryRemoveRule.onMatch(HiveSubQueryRemoveRule.java:115) at org.apache.calcite.plan.AbstractRelOptPlanner.fireRule(AbstractRelOptPlanner.java:319) at org.apache.calcite.plan.hep.HepPlanner.applyRule(HepPlanner.java:560) at org.apache.calcite.plan.hep.HepPlanner.applyRules(HepPlanner.java:419) at org.apache.calcite.plan.hep.HepPlanner.executeInstruction(HepPlanner.java:256) at org.apache.calcite.plan.hep.HepInstruction$RuleInstance.execute(HepInstruction.java:127) at org.apache.calcite.plan.hep.HepPlanner.executeProgram(HepPlanner.java:215) at org.apache.calcite.plan.hep.HepPlanner.findBestExp(HepPlanner.java:202) at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2518) at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.executeProgram(CalcitePlanner.java:2484) at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.hepPlan(CalcitePlanner.java:2459) at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1814) at org.apache.hadoop.hive.ql.parse.CalcitePlanner$CalcitePlannerAction.apply(CalcitePlanner.java:1731) at org.apache.calcite.tools.Frameworks.lambda$withPlanner$0(Frameworks.java:130) at org.apache.calcite.tools.Frameworks$$Lambda$162/385313421.apply(Unknown Source) at org.apache.calcite.prepare.CalcitePrepareImpl.perform(CalcitePrepareImpl.java:915) at org.apache.calcite.tools.Frameworks.withPrepare(Frameworks.java:179) at org.apache.calcite.tools.Frameworks.withPlanner(Frameworks.java:125) at org.apache.hadoop.hive.ql.parse.CalcitePlanner.logicalPlan(CalcitePlanner.java:1492) at org.apache.hadoop.hive.ql.parse.CalcitePlanner.genOPTree(CalcitePlanner.java:468) at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:12488) at org.apache.hadoop.hive.ql.parse.CalcitePlanner.analyzeInternal(CalcitePlanner.java:358) at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283) at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:171) at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:283) at org.apache.hadoop.hive.ql.Compiler.analyze(Compiler.java:219) at org.apache.hadoop.hive.ql.Compiler.compile(Compiler.java:103) at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:215) at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:828) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:774) at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:768) at org.apache.hadoop.hive.ql.reexec.ReExecDriver.compileAndRespond(ReExecDriver.java:125) at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:203) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:325) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1688) at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:345) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)