[DRILL-843] Drillbit goes down after running a large number of queries in a series

• Type: Bug
• Status: Closed
• Priority: Blocker
• Resolution: Fixed
• Affects Version/s: 0.4.0
• Fix Version/s: 0.4.0
• Component/s: Execution - Flow
• Assignee: Aditya Kishore
• Reporter: Ramana Inukonda Nagaraj
• Created: 28/May/14 00:54
• Updated: 17/Apr/15 18:20
• Resolved: 28/May/14 19:27

Description

I am not sure where this goes so I put it under Execution.
After a large number of queries run against the same cluster I had a drillbit go down and fail my tests.

This is the log in drillbit.out on the node where the drillbit went down

Exception in thread "WorkManager-4" java.lang.OutOfMemoryError: PermGen space
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.lang.ClassLoader.defineClass(ClassLoader.java:643)
at org.apache.drill.exec.compile.QueryClassLoader.findClass(QueryClassLoader.java:61)
at org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:294)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:185)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:179)
at org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec.createPartitioner(PartitionSenderRootExec.java:318)
at org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec.next(PartitionSenderRootExec.java:136)
at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:98)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Exception in thread "WorkManager-11" java.lang.OutOfMemoryError: PermGen space
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.lang.ClassLoader.defineClass(ClassLoader.java:643)
at org.apache.drill.exec.compile.QueryClassLoader.findClass(QueryClassLoader.java:61)
at org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:294)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:185)
at org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.createNewSorter(ExternalSortBatch.java:449)
at org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.next(ExternalSortBatch.java:204)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:63)
at org.apache.drill.exec.record.AbstractSingleRecordBatch.next(AbstractSingleRecordBatch.java:47)
at org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.next(RemovingRecordBatch.java:94)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:63)
at org.apache.drill.exec.record.AbstractSingleRecordBatch.next(AbstractSingleRecordBatch.java:47)
at org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.next(ProjectRecordBatch.java:83)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)
at org.apache.drill.exec.physical.impl.join.JoinStatus.nextRight(JoinStatus.java:74)
at org.apache.drill.exec.physical.impl.join.JoinStatus.ensureInitial(JoinStatus.java:80)
at org.apache.drill.exec.physical.impl.join.MergeJoinBatch.next(MergeJoinBatch.java:137)
at org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec.next(PartitionSenderRootExec.java:102)
at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:98)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Exception in thread "WorkManager-8" java.lang.OutOfMemoryError: PermGen space
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.lang.ClassLoader.defineClass(ClassLoader.java:643)
at org.apache.drill.exec.compile.QueryClassLoader.findClass(QueryClassLoader.java:61)
at org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:294)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:185)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:179)
at org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec.createPartitioner(PartitionSenderRootExec.java:318)
at org.apache.drill.exec.physical.impl.partitionsender.PartitionSenderRootExec.next(PartitionSenderRootExec.java:136)
at org.apache.drill.exec.work.fragment.FragmentExecutor.run(FragmentExecutor.java:98)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Exception in thread "WorkManager-15" java.lang.OutOfMemoryError: PermGen space
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at java.lang.ClassLoader.defineClass(ClassLoader.java:643)
at org.apache.drill.exec.compile.QueryClassLoader.findClass(QueryClassLoader.java:61)
at org.apache.drill.exec.compile.ClassTransformer.getImplementationClass(ClassTransformer.java:294)
at org.apache.drill.exec.ops.FragmentContext.getImplementationClass(FragmentContext.java:185)
at org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.createNewSorter(ExternalSortBatch.java:449)
at org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.next(ExternalSortBatch.java:204)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:63)
at org.apache.drill.exec.record.AbstractSingleRecordBatch.next(AbstractSingleRecordBatch.java:47)
at org.apache.drill.exec.physical.impl.svremover.RemovingRecordBatch.next(RemovingRecordBatch.java:94)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:63)
at org.apache.drill.exec.record.AbstractSingleRecordBatch.next(AbstractSingleRecordBatch.java:47)
at org.apache.drill.exec.physical.impl.project.ProjectRecordBatch.next(ProjectRecordBatch.java:83)
at org.apache.drill.exec.record.AbstractRecordBatch.next(AbstractRecordBatch.java:69)