JVM version is 24.79-b02 Deadlock Detection: Found one Java-level deadlock: ============================= "Thread-403": waiting to lock Monitor@0x00007f15186bf278 (Object@0x00000000d49c11f0, a java/util/Properties), which is held by "Thread-404" "Thread-404": waiting to lock Monitor@0x00007f151001e008 (Object@0x00000000d49c1238, a java/util/Properties), which is held by "Thread-403" Found a total of 1 deadlock. Thread 10015: (state = BLOCKED) - java.util.Hashtable.get(java.lang.Object) @bci=0, line=433 (Compiled frame) - java.util.Hashtable.equals(java.lang.Object) @bci=118, line=876 (Interpreted frame) - org.apache.hadoop.hive.ql.plan.TableDesc.equals(java.lang.Object) @bci=127, line=196 (Interpreted frame) - org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(org.apache.hadoop.mapred.JobConf, int) @bci=327, line=332 (Interpreted frame) - org.apache.hadoop.hive.ql.io.CombineHiveInputFormat.getSplits(org.apache.hadoop.mapred.JobConf, int) @bci=266, line=316 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.writeOldSplits(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path) @bci=9, line=520 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.fs.Path) @bci=31, line=512 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(org.apache.hadoop.mapreduce.Job, org.apache.hadoop.mapreduce.Cluster) @bci=320, line=394 (Interpreted frame) - org.apache.hadoop.mapreduce.Job$10.run() @bci=15, line=1285 (Interpreted frame) - org.apache.hadoop.mapreduce.Job$10.run() @bci=1, line=1282 (Interpreted frame) - java.security.AccessController.doPrivileged(java.security.PrivilegedExceptionAction, java.security.AccessControlContext) @bci=0 (Interpreted frame) - javax.security.auth.Subject.doAs(javax.security.auth.Subject, java.security.PrivilegedExceptionAction) @bci=42, line=415 (Interpreted frame) - org.apache.hadoop.security.UserGroupInformation.doAs(java.security.PrivilegedExceptionAction) @bci=14, line=1594 (Interpreted frame) - org.apache.hadoop.mapreduce.Job.submit() @bci=48, line=1282 (Interpreted frame) - org.apache.hadoop.mapred.JobClient$1.run() @bci=9, line=562 (Interpreted frame) - org.apache.hadoop.mapred.JobClient$1.run() @bci=1, line=557 (Interpreted frame) - java.security.AccessController.doPrivileged(java.security.PrivilegedExceptionAction, java.security.AccessControlContext) @bci=0 (Interpreted frame) - javax.security.auth.Subject.doAs(javax.security.auth.Subject, java.security.PrivilegedExceptionAction) @bci=42, line=415 (Interpreted frame) - org.apache.hadoop.security.UserGroupInformation.doAs(java.security.PrivilegedExceptionAction) @bci=14, line=1594 (Interpreted frame) - org.apache.hadoop.mapred.JobClient.submitJobInternal(org.apache.hadoop.mapred.JobConf) @bci=27, line=557 (Interpreted frame) - org.apache.hadoop.mapred.JobClient.submitJob(org.apache.hadoop.mapred.JobConf) @bci=2, line=548 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(org.apache.hadoop.hive.ql.DriverContext) @bci=1511, line=420 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(org.apache.hadoop.hive.ql.DriverContext) @bci=406, line=136 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.Task.executeTask() @bci=30, line=153 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() @bci=7, line=85 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.TaskRunner.run() @bci=16, line=72 (Interpreted frame) Thread 10014: (state = BLOCKED) - java.util.Hashtable.size() @bci=0, line=306 (Interpreted frame) - java.util.Hashtable.equals(java.lang.Object) @bci=22, line=863 (Interpreted frame) - org.apache.hadoop.hive.ql.plan.TableDesc.equals(java.lang.Object) @bci=127, line=196 (Interpreted frame) - org.apache.hadoop.hive.ql.io.HiveInputFormat.getSplits(org.apache.hadoop.mapred.JobConf, int) @bci=327, line=332 (Interpreted frame) - org.apache.hadoop.hive.ql.io.CombineHiveInputFormat.getSplits(org.apache.hadoop.mapred.JobConf, int) @bci=266, line=316 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.writeOldSplits(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.fs.Path) @bci=9, line=520 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(org.apache.hadoop.mapreduce.JobContext, org.apache.hadoop.fs.Path) @bci=31, line=512 (Interpreted frame) - org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(org.apache.hadoop.mapreduce.Job, org.apache.hadoop.mapreduce.Cluster) @bci=320, line=394 (Interpreted frame) - org.apache.hadoop.mapreduce.Job$10.run() @bci=15, line=1285 (Interpreted frame) - org.apache.hadoop.mapreduce.Job$10.run() @bci=1, line=1282 (Interpreted frame) - java.security.AccessController.doPrivileged(java.security.PrivilegedExceptionAction, java.security.AccessControlContext) @bci=0 (Interpreted frame) - javax.security.auth.Subject.doAs(javax.security.auth.Subject, java.security.PrivilegedExceptionAction) @bci=42, line=415 (Interpreted frame) - org.apache.hadoop.security.UserGroupInformation.doAs(java.security.PrivilegedExceptionAction) @bci=14, line=1594 (Interpreted frame) - org.apache.hadoop.mapreduce.Job.submit() @bci=48, line=1282 (Interpreted frame) - org.apache.hadoop.mapred.JobClient$1.run() @bci=9, line=562 (Interpreted frame) - org.apache.hadoop.mapred.JobClient$1.run() @bci=1, line=557 (Interpreted frame) - java.security.AccessController.doPrivileged(java.security.PrivilegedExceptionAction, java.security.AccessControlContext) @bci=0 (Interpreted frame) - javax.security.auth.Subject.doAs(javax.security.auth.Subject, java.security.PrivilegedExceptionAction) @bci=42, line=415 (Interpreted frame) - org.apache.hadoop.security.UserGroupInformation.doAs(java.security.PrivilegedExceptionAction) @bci=14, line=1594 (Interpreted frame) - org.apache.hadoop.mapred.JobClient.submitJobInternal(org.apache.hadoop.mapred.JobConf) @bci=27, line=557 (Interpreted frame) - org.apache.hadoop.mapred.JobClient.submitJob(org.apache.hadoop.mapred.JobConf) @bci=2, line=548 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(org.apache.hadoop.hive.ql.DriverContext) @bci=1511, line=420 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(org.apache.hadoop.hive.ql.DriverContext) @bci=406, line=136 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.Task.executeTask() @bci=30, line=153 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() @bci=7, line=85 (Interpreted frame) - org.apache.hadoop.hive.ql.exec.TaskRunner.run() @bci=16, line=72 (Interpreted frame) Thread 9963: (state = IN_NATIVE) - org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(int, org.apache.hadoop.net.unix.DomainSocketWatcher$FdSet) @bci=0 (Interpreted frame) - org.apache.hadoop.net.unix.DomainSocketWatcher.access$800(int, org.apache.hadoop.net.unix.DomainSocketWatcher$FdSet) @bci=2, line=52 (Interpreted frame) - org.apache.hadoop.net.unix.DomainSocketWatcher$1.run() @bci=551, line=457 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=745 (Interpreted frame) Thread 9960: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.park(java.lang.Object) @bci=14, line=186 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await() @bci=42, line=2043 (Interpreted frame) - java.util.concurrent.ArrayBlockingQueue.take() @bci=20, line=374 (Interpreted frame) - com.jolbox.bonecp.PoolWatchThread.run() @bci=93, line=75 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.runWorker(java.util.concurrent.ThreadPoolExecutor$Worker) @bci=95, line=1145 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=5, line=615 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=745 (Interpreted frame) Thread 9959: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=226 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(long) @bci=68, line=2082 (Interpreted frame) - java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=122, line=1090 (Interpreted frame) - java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=1, line=807 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.getTask() @bci=156, line=1068 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.runWorker(java.util.concurrent.ThreadPoolExecutor$Worker) @bci=26, line=1130 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=5, line=615 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=745 (Interpreted frame) Thread 9958: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=135 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove() @bci=2, line=151 (Interpreted frame) - com.google.common.base.internal.Finalizer.run() @bci=5, line=127 (Interpreted frame) Thread 9956: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.park(java.lang.Object) @bci=14, line=186 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await() @bci=42, line=2043 (Interpreted frame) - java.util.concurrent.ArrayBlockingQueue.take() @bci=20, line=374 (Interpreted frame) - com.jolbox.bonecp.PoolWatchThread.run() @bci=93, line=75 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.runWorker(java.util.concurrent.ThreadPoolExecutor$Worker) @bci=95, line=1145 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=5, line=615 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=745 (Interpreted frame) Thread 9955: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=226 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(long) @bci=68, line=2082 (Interpreted frame) - java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=122, line=1090 (Interpreted frame) - java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take() @bci=1, line=807 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.getTask() @bci=156, line=1068 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.runWorker(java.util.concurrent.ThreadPoolExecutor$Worker) @bci=26, line=1130 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=5, line=615 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=745 (Interpreted frame) Thread 9954: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=135 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove() @bci=2, line=151 (Interpreted frame) - com.google.common.base.internal.Finalizer.run() @bci=5, line=127 (Interpreted frame) Thread 9953: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Compiled frame; information may be imprecise) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=135 (Compiled frame) - com.mysql.jdbc.AbandonedConnectionCleanupThread.run() @bci=16, line=43 (Compiled frame) Thread 9844: (state = BLOCKED) Thread 9843: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=135 (Compiled frame) - java.lang.ref.ReferenceQueue.remove() @bci=2, line=151 (Compiled frame) - java.lang.ref.Finalizer$FinalizerThread.run() @bci=36, line=209 (Compiled frame) Thread 9842: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.Object.wait() @bci=2, line=503 (Compiled frame) - java.lang.ref.Reference$ReferenceHandler.run() @bci=46, line=133 (Compiled frame) Thread 9827: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Compiled frame; information may be imprecise) - org.apache.hadoop.hive.ql.DriverContext.pollFinished() @bci=62, line=118 (Interpreted frame) - org.apache.hadoop.hive.ql.Driver.execute() @bci=872, line=1282 (Interpreted frame) - org.apache.hadoop.hive.ql.Driver.runInternal(java.lang.String, boolean) @bci=639, line=1093 (Interpreted frame) - org.apache.hadoop.hive.ql.Driver.run(java.lang.String, boolean) @bci=3, line=916 (Interpreted frame) - org.apache.hadoop.hive.ql.Driver.run(java.lang.String) @bci=3, line=906 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner.processQueryCmd(org.apache.hadoop.hive.ql.processors.CommandProcessor, com.treasure_data.hadoop.hive.runner.CliSessionState, org.apache.hadoop.hive.ql.session.SessionState$LogHelper, java.lang.String) @bci=104, line=448 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner.processCmd(java.lang.String) @bci=212, line=389 (Interpreted frame) - org.apache.hadoop.hive.cli.CliDriver.processLine(java.lang.String, boolean) @bci=153, line=423 (Interpreted frame) - org.apache.hadoop.hive.cli.CliDriver.processLine(java.lang.String) @bci=3, line=359 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner.run(com.treasure_data.hadoop.hive.runner.CliSessionState) @bci=410, line=309 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner$1.run() @bci=8, line=191 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner$1.run() @bci=1, line=189 (Interpreted frame) - java.security.AccessController.doPrivileged(java.security.PrivilegedExceptionAction, java.security.AccessControlContext) @bci=0 (Interpreted frame) - javax.security.auth.Subject.doAs(javax.security.auth.Subject, java.security.PrivilegedExceptionAction) @bci=42, line=415 (Interpreted frame) - org.apache.hadoop.security.UserGroupInformation.doAs(java.security.PrivilegedExceptionAction) @bci=14, line=1594 (Interpreted frame) - com.treasure_data.hadoop.util.TDUtil.doAs(java.lang.String, java.security.PrivilegedExceptionAction) @bci=7, line=226 (Interpreted frame) - com.treasure_data.hadoop.hive.runner.QueryRunner.main(java.lang.String[]) @bci=283, line=189 (Interpreted frame) - sun.reflect.NativeMethodAccessorImpl.invoke0(java.lang.reflect.Method, java.lang.Object, java.lang.Object[]) @bci=0 (Interpreted frame) - sun.reflect.NativeMethodAccessorImpl.invoke(java.lang.Object, java.lang.Object[]) @bci=87, line=57 (Interpreted frame) - sun.reflect.DelegatingMethodAccessorImpl.invoke(java.lang.Object, java.lang.Object[]) @bci=6, line=43 (Interpreted frame) - java.lang.reflect.Method.invoke(java.lang.Object, java.lang.Object[]) @bci=57, line=606 (Interpreted frame) - org.apache.hadoop.util.RunJar.main(java.lang.String[]) @bci=604, line=212 (Interpreted frame)