"regionserver60020-largeCompactions-1392787537634" daemon prio=10 tid=0x0000000001580800 nid=0x25af runnable [0x0000000059def000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.readBytes(Native Method) at java.io.FileInputStream.read(FileInputStream.java:272) at org.apache.hadoop.hdfs.BlockReaderLocal.read(BlockReaderLocal.java:413) - locked <0x000000023f703658> (a org.apache.hadoop.hdfs.BlockReaderLocal) at org.apache.hadoop.hdfs.DFSInputStream$ByteArrayStrategy.doRead(DFSInputStream.java:691) at org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:747) - eliminated <0x0000000569d166a8> (a org.apache.hadoop.hdfs.DFSInputStream) at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:801) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:842) - locked <0x0000000569d166a8> (a org.apache.hadoop.hdfs.DFSInputStream) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.hbase.io.hfile.HFileBlock.readWithExtra(HFileBlock.java:584) at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1414) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockDataInternal(HFileBlock.java:1859) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderV2.readBlockData(HFileBlock.java:1703) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:347) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.readNextDataBlock(HFileReaderV2.java:606) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$EncodedScannerV2.next(HFileReaderV2.java:1033) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:125) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:95) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:383) - locked <0x00000003e5635c70> (a org.apache.hadoop.hbase.regionserver.StoreScanner) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:311) - locked <0x00000003e5635c70> (a org.apache.hadoop.hbase.regionserver.StoreScanner) at org.apache.hadoop.hbase.regionserver.Compactor.compact(Compactor.java:197) at org.apache.hadoop.hbase.regionserver.Store.compact(Store.java:1150) at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1403) at org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.run(CompactionRequest.java:303) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) Locked ownable synchronizers: - <0x00000002be65fb30> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00000002e1a1f380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "IPC Server handler 255 on 60020" daemon prio=10 tid=0x00007f10c41ad800 nid=0x17b1 waiting on condition [0x0000000054c9e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004054ecbf0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:873) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5945) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5915) at org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:5843) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1842) at org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1828) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4866) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4840) at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2279) at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:315) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1445) Locked ownable synchronizers: - None 1 threads with trace: States: {TIMED_WAITING=1} Stack: - sun.misc.Unsafe.park(Native Method) - - parking to wait for (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) - java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) - java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) - java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:873) - org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5945) - org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5915) - org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:5843) - org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3889) - - locked (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl) - org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3940) - org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4867) - org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4840) - org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2279) - sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source) - sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - java.lang.reflect.Method.invoke(Method.java:606) - org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:315) - org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1445) "IPC Server handler 251 on 60020" daemon prio=10 tid=0x00007f10c41a5800 nid=0x17ad waiting on condition [0x000000005489a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004054ecbf0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:873) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5945) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5915) at org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:5843) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2201) at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3910) at sun.reflect.GeneratedMethodAccessor29.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:315) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1445) Locked ownable synchronizers: - None "IPC Server handler 199 on 60020" daemon prio=10 tid=0x00007f10c413a000 nid=0x175e waiting on condition [0x0000000051466000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004054ecbf0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1033) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1326) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:873) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5945) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5915) at org.apache.hadoop.hbase.regionserver.HRegion.startRegionOperation(HRegion.java:5843) at org.apache.hadoop.hbase.regionserver.HRegionServer.next(HRegionServer.java:2734) at sun.reflect.GeneratedMethodAccessor33.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:315) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1445) Locked ownable synchronizers: - None "IPC Server handler 63 on 60020" daemon prio=10 tid=0x00007f10c4022000 nid=0x16d4 waiting on condition [0x0000000048bde000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000004054ecbf0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireNanos(AbstractQueuedSynchronizer.java:929) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireNanos(AbstractQueuedSynchronizer.java:1245) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.tryLock(ReentrantReadWriteLock.java:1115) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5945) at org.apache.hadoop.hbase.regionserver.HRegion.lock(HRegion.java:5915) at org.apache.hadoop.hbase.regionserver.HRegion.startBulkRegionOperation(HRegion.java:5874) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:3665) at org.apache.hadoop.hbase.regionserver.HRegion.bulkLoadHFiles(HRegion.java:3648) at org.apache.hadoop.hbase.regionserver.HRegionServer.bulkLoadHFiles(HRegionServer.java:3028) at sun.reflect.GeneratedMethodAccessor67.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:315) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1445) Locked ownable synchronizers: - None