2014-05-29 08:57:38 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.51-b03 mixed mode): "LoadThread-0" prio=6 tid=0x0000000012c06800 nid=0xc04 waiting for monitor entry [0x0000000014c2e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.jackrabbit.oak.plugins.segment.file.FileStore.writeSegment(FileStore.java:610) - waiting to lock <0x0000000704bcaf08> (a org.apache.jackrabbit.oak.plugins.segment.file.FileStore) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.flush(SegmentWriter.java:213) - locked <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.prepare(SegmentWriter.java:271) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.prepare(SegmentWriter.java:237) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeValueRecord(SegmentWriter.java:512) - locked <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.internalWriteStream(SegmentWriter.java:765) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeStream(SegmentWriter.java:747) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeBlob(SegmentWriter.java:717) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeProperty(SegmentWriter.java:808) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeProperty(SegmentWriter.java:796) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:1004) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeAdded(SegmentWriter.java:967) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:393) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:992) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeChanged(SegmentWriter.java:973) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:396) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeChanged(SegmentWriter.java:973) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:396) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:61) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:22) at org.apache.jackrabbit.oak.spi.commit.EditorHook.processCommit(EditorHook.java:57) at org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:60) at org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:60) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.prepare(SegmentNodeStore.java:299) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.optimisticMerge(SegmentNodeStore.java:330) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.execute(SegmentNodeStore.java:386) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore.merge(SegmentNodeStore.java:133) at org.apache.jackrabbit.oak.core.MutableRoot.commit(MutableRoot.java:247) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.commit(SessionDelegate.java:333) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.save(SessionDelegate.java:479) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.perform(SessionImpl.java:417) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.perform(SessionImpl.java:414) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.perform(SessionDelegate.java:236) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.perform(SessionImpl.java:127) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.save(SessionImpl.java:414) at org.apache.jackrabbit.oak.scalability.ScalabilityBlobSearchSuite$Writer.run(ScalabilityBlobSearchSuite.java:457) at java.lang.Thread.run(Thread.java:744) "pool-1-thread-1" prio=6 tid=0x0000000013ba2800 nid=0x3738 waiting on condition [0x00000000149bf000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000704f68200> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) "TarMK flush thread: target\Oak-Tar-FDS-1401333983739" daemon prio=2 tid=0x00000000114d7800 nid=0x37ec waiting for monitor entry [0x00000000127ee000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.dropCache(SegmentWriter.java:731) - waiting to lock <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.file.FileStore.flush(FileStore.java:404) - locked <0x0000000704bcaf08> (a org.apache.jackrabbit.oak.plugins.segment.file.FileStore) - locked <0x0000000704d66fa0> (a java.util.concurrent.atomic.AtomicReference) at org.apache.jackrabbit.oak.plugins.segment.file.FileStore$1.run(FileStore.java:247) at java.lang.Thread.run(Thread.java:744) "Service Thread" daemon prio=6 tid=0x000000000f562000 nid=0x3168 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x000000000f560000 nid=0x2930 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x000000000f553000 nid=0x201c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Attach Listener" daemon prio=10 tid=0x000000000f552000 nid=0x3798 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x000000000f54b000 nid=0x316c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=8 tid=0x000000000f4f8000 nid=0x266c in Object.wait() [0x00000000107af000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000704baf068> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x0000000704baf068> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) "Reference Handler" daemon prio=10 tid=0x000000000f4ef000 nid=0x35b8 in Object.wait() [0x000000001065e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000704b9c9b8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x0000000704b9c9b8> (a java.lang.ref.Reference$Lock) "main" prio=6 tid=0x000000000238f800 nid=0x2e00 in Object.wait() [0x000000000272f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x000000070511ff58> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1280) - locked <0x000000070511ff58> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1354) at org.apache.jackrabbit.oak.scalability.ScalabilityBlobSearchSuite.createLoad(ScalabilityBlobSearchSuite.java:288) at org.apache.jackrabbit.oak.scalability.ScalabilityBlobSearchSuite.beforeIteration(ScalabilityBlobSearchSuite.java:248) at org.apache.jackrabbit.oak.scalability.ScalabilityAbstractSuite.setupIteration(ScalabilityAbstractSuite.java:187) at org.apache.jackrabbit.oak.scalability.ScalabilityAbstractSuite.runSuite(ScalabilityAbstractSuite.java:150) at org.apache.jackrabbit.oak.scalability.ScalabilityAbstractSuite.run(ScalabilityAbstractSuite.java:125) at org.apache.jackrabbit.oak.scalability.ScalabilityRunner.main(ScalabilityRunner.java:185) at org.apache.jackrabbit.oak.run.Main.main(Main.java:122) "VM Thread" prio=10 tid=0x000000000f4eb800 nid=0x368c runnable "GC task thread#0 (ParallelGC)" prio=6 tid=0x00000000021ed800 nid=0x29e8 runnable "GC task thread#1 (ParallelGC)" prio=6 tid=0x00000000021ef800 nid=0x29b8 runnable "GC task thread#2 (ParallelGC)" prio=6 tid=0x00000000021f1000 nid=0x3668 runnable "GC task thread#3 (ParallelGC)" prio=6 tid=0x00000000021f2800 nid=0x2fe0 runnable "VM Periodic Task Thread" prio=10 tid=0x000000000f56b800 nid=0x3488 waiting on condition JNI global references: 288 Found one Java-level deadlock: ============================= "LoadThread-0": waiting to lock monitor 0x0000000011efe358 (object 0x0000000704bcaf08, a org.apache.jackrabbit.oak.plugins.segment.file.FileStore), which is held by "TarMK flush thread: target\Oak-Tar-FDS-1401333983739" "TarMK flush thread: target\Oak-Tar-FDS-1401333983739": waiting to lock monitor 0x0000000011efe2a8 (object 0x0000000704d97c18, a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter), which is held by "LoadThread-0" Java stack information for the threads listed above: =================================================== "LoadThread-0": at org.apache.jackrabbit.oak.plugins.segment.file.FileStore.writeSegment(FileStore.java:610) - waiting to lock <0x0000000704bcaf08> (a org.apache.jackrabbit.oak.plugins.segment.file.FileStore) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.flush(SegmentWriter.java:213) - locked <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.prepare(SegmentWriter.java:271) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.prepare(SegmentWriter.java:237) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeValueRecord(SegmentWriter.java:512) - locked <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.internalWriteStream(SegmentWriter.java:765) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeStream(SegmentWriter.java:747) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeBlob(SegmentWriter.java:717) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeProperty(SegmentWriter.java:808) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeProperty(SegmentWriter.java:796) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:1004) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeAdded(SegmentWriter.java:967) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:393) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:992) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeChanged(SegmentWriter.java:973) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:396) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter$2.childNodeChanged(SegmentWriter.java:973) at org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:396) at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.writeNode(SegmentWriter.java:964) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:61) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:22) at org.apache.jackrabbit.oak.spi.commit.EditorHook.processCommit(EditorHook.java:57) at org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:60) at org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:60) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.prepare(SegmentNodeStore.java:299) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.optimisticMerge(SegmentNodeStore.java:330) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore$Commit.execute(SegmentNodeStore.java:386) at org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore.merge(SegmentNodeStore.java:133) at org.apache.jackrabbit.oak.core.MutableRoot.commit(MutableRoot.java:247) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.commit(SessionDelegate.java:333) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.save(SessionDelegate.java:479) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.perform(SessionImpl.java:417) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.perform(SessionImpl.java:414) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.perform(SessionDelegate.java:236) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.perform(SessionImpl.java:127) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.save(SessionImpl.java:414) at org.apache.jackrabbit.oak.scalability.ScalabilityBlobSearchSuite$Writer.run(ScalabilityBlobSearchSuite.java:457) at java.lang.Thread.run(Thread.java:744) "TarMK flush thread: target\Oak-Tar-FDS-1401333983739": at org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.dropCache(SegmentWriter.java:731) - waiting to lock <0x0000000704d97c18> (a org.apache.jackrabbit.oak.plugins.segment.SegmentWriter) at org.apache.jackrabbit.oak.plugins.segment.file.FileStore.flush(FileStore.java:404) - locked <0x0000000704bcaf08> (a org.apache.jackrabbit.oak.plugins.segment.file.FileStore) - locked <0x0000000704d66fa0> (a java.util.concurrent.atomic.AtomicReference) at org.apache.jackrabbit.oak.plugins.segment.file.FileStore$1.run(FileStore.java:247) at java.lang.Thread.run(Thread.java:744) Found 1 deadlock.