2012-05-06 21:09:58 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "IPC Client (576300853) connection to /10.18.40.217:9000 from root" daemon prio=10 tid=0x0000000040119000 nid=0x45c8 in Object.wait() [0x00007f7c69caa000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:733) - locked <0x00007f7c83d18790> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:776) "regionserver60020-smallCompactions-1336337886145" daemon prio=10 tid=0x00007f7c48e98000 nid=0x4221 waiting on condition [0x00007f7c55ddc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "regionserver60020-splits-1336337850049" daemon prio=10 tid=0x0000000042586800 nid=0x415c waiting on condition [0x00007f7c572f1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390f38> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-31" prio=10 tid=0x0000000040951800 nid=0x7243 waiting on condition [0x00007f7c57dfc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-30" prio=10 tid=0x00000000409ae800 nid=0x7242 waiting on condition [0x00007f7c57efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-29" prio=10 tid=0x00007f7c58038800 nid=0x7239 waiting on condition [0x00007f7c56eed000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-28" prio=10 tid=0x0000000040aa3000 nid=0x722c waiting on condition [0x00007f7c57bfa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-27" prio=10 tid=0x00000000409ae000 nid=0x7210 waiting on condition [0x00007f7c5c531000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-26" prio=10 tid=0x0000000040ac4000 nid=0x7204 waiting on condition [0x00007f7c579f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-25" prio=10 tid=0x0000000040ac1800 nid=0x7203 waiting on condition [0x00007f7c57af9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-24" prio=10 tid=0x000000004094a000 nid=0x71fa waiting on condition [0x00007f7c5c12d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-23" prio=10 tid=0x00007f7c60b5d800 nid=0x71f9 waiting on condition [0x00007f7c5c22e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-22" prio=10 tid=0x0000000040947000 nid=0x71f8 waiting on condition [0x00007f7c5c32f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-21" prio=10 tid=0x000000004093d800 nid=0x71f6 waiting on condition [0x00007f7c5c430000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-20" prio=10 tid=0x0000000040449800 nid=0x71f3 waiting on condition [0x00007f7c5c632000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-19" prio=10 tid=0x00007f7c58019000 nid=0x71f1 waiting on condition [0x00007f7c5c733000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-18" prio=10 tid=0x000000004084e800 nid=0x71f0 waiting on condition [0x00007f7c5c834000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-17" prio=10 tid=0x000000004084d800 nid=0x71ef waiting on condition [0x00007f7c5c935000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-16" prio=10 tid=0x0000000040849000 nid=0x71ee waiting on condition [0x00007f7c5ca36000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-15" prio=10 tid=0x0000000040a8b800 nid=0x71ed waiting on condition [0x00007f7c5cb37000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-14" prio=10 tid=0x00007f7c60b5c000 nid=0x71ec waiting on condition [0x00007f7c5cc38000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-13" prio=10 tid=0x0000000040a88800 nid=0x71eb waiting on condition [0x00007f7c5cd39000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-12" prio=10 tid=0x0000000040a86000 nid=0x71ea waiting on condition [0x00007f7c5ce3a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-11" prio=10 tid=0x0000000040a7e800 nid=0x71e7 waiting on condition [0x00007f7c5cf3b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-10" prio=10 tid=0x0000000040485800 nid=0x71e1 waiting on condition [0x00007f7c5d03c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-9" prio=10 tid=0x0000000040485000 nid=0x71e0 waiting on condition [0x00007f7c5d23e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-8" prio=10 tid=0x0000000040484800 nid=0x71df waiting on condition [0x00007f7c5d33f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-7" prio=10 tid=0x000000004099c800 nid=0x71de waiting on condition [0x00007f7c5d13d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-6" prio=10 tid=0x000000004099b800 nid=0x71dd waiting on condition [0x00007f7c5d844000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-5" prio=10 tid=0x00007f7c602bb000 nid=0x71d9 waiting on condition [0x00007f7c69ba9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-4" prio=10 tid=0x0000000040479800 nid=0x6ffc waiting on condition [0x00007f7c5d642000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-3" prio=10 tid=0x0000000040db3000 nid=0x6ff6 waiting on condition [0x00007f7c5d440000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-2" prio=10 tid=0x0000000040c13000 nid=0x6fec waiting on condition [0x00007f7c5da46000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "PostOpenDeployTasks:09b9a6c12abee4fe621c159bbe78852b-EventThread" daemon prio=10 tid=0x0000000040a5b800 nid=0x6fdd waiting on condition [0x00007f7c5dddc000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7ca070f288> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "PostOpenDeployTasks:09b9a6c12abee4fe621c159bbe78852b-SendThread(HOST-192-168-47-205:2181)" daemon prio=10 tid=0x0000000040a53000 nid=0x6fdc runnable [0x00007f7c5d945000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00007f7ca06ea1c0> (a sun.nio.ch.Util$1) - locked <0x00007f7ca06ea1d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007f7c9eb649f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-1" prio=10 tid=0x000000004036b000 nid=0x6fbc waiting on condition [0x00007f7c5db47000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "RS_OPEN_REGION-HOST-192-168-47-207,60020,1336336462275-0" prio=10 tid=0x0000000040368000 nid=0x6fbb waiting on condition [0x00007f7c5dc48000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9f390cb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "regionserver60020.compactionChecker" daemon prio=10 tid=0x00007f7c6c908800 nid=0x6f1d in Object.wait() [0x00007f7c694a2000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00007f7c9eb200d0> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) - locked <0x00007f7c9eb200d0> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:75) at java.lang.Thread.run(Thread.java:619) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "regionserver60020.logSyncer" daemon prio=10 tid=0x0000000040b29800 nid=0x6f19 runnable [0x00007f7c698a6000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:302) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1288) - locked <0x00007f7c9f39b758> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.wal.HLog.syncer(HLog.java:1240) at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:1393) at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1195) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "regionserver60020-EventThread" daemon prio=10 tid=0x0000000040abc000 nid=0x6f14 waiting on condition [0x00007f7c69dab000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) "regionserver60020-SendThread(HOST-192-168-47-204:2181)" daemon prio=10 tid=0x0000000040a04800 nid=0x6f13 runnable [0x00007f7c69eac000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:215) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00007f7c9eb4be80> (a sun.nio.ch.Util$1) - locked <0x00007f7c9eb4be98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007f7c9eb4a710> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "regionserver60020" prio=10 tid=0x0000000040a35800 nid=0x6f11 waiting on condition [0x00007f7c6a1b0000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.mortbay.thread.QueuedThreadPool.doStop(QueuedThreadPool.java:435) at org.mortbay.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:76) - locked <0x00007f7c9f268400> (a java.lang.Object) at org.mortbay.jetty.Server.doStop(Server.java:291) at org.mortbay.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:76) - locked <0x00007f7c9f2683b8> (a java.lang.Object) at org.apache.hadoop.http.HttpServer.stop(HttpServer.java:683) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:735) at java.lang.Thread.run(Thread.java:619) "LRU Statistics #0" daemon prio=10 tid=0x0000000040a38800 nid=0x6f10 waiting on condition [0x00007f7c6a2b1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b698> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "main.LruBlockCache.EvictionThread" daemon prio=10 tid=0x0000000040a37800 nid=0x6f0f in Object.wait() [0x00007f7c701dd000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:594) - locked <0x00007f7c9eb0b6e8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) at java.lang.Thread.run(Thread.java:619) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4077 2012-05-06 21:13:20 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4145 2012-05-06 21:13:22 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4145 2012-05-06 21:13:24 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4145 2012-05-06 21:14:19 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4147 2012-05-06 21:16:25 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4151 2012-05-06 21:19:07 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4157 2012-05-06 21:28:13 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4175 2012-05-06 21:38:18 Full thread dump Java HotSpot(TM) 64-Bit Server VM (14.3-b01 mixed mode): "Attach Listener" daemon prio=10 tid=0x0000000040fd7800 nid=0x45e0 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "1997531088@qtp-716941846-5" prio=10 tid=0x00007f7c5820c800 nid=0xe1b in Object.wait() [0x00007f7c56ae8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfe0616d0> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe74d758> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfca02fc0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "1374615312@qtp-716941846-3" prio=10 tid=0x00007f7c58214800 nid=0xc42 in Object.wait() [0x00007f7c55bd9000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7cfdbb6cc8> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7cfe7dbda0> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7cfe30ebe0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "615294984@qtp-716941846-2" prio=10 tid=0x00007f7c58086000 nid=0x907 in Object.wait() [0x00007f7c57ffd000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.io.nio.SelectChannelEndPoint.blockWritable(SelectChannelEndPoint.java:279) - locked <0x00007f7d6383cd40> (a org.mortbay.jetty.nio.SelectChannelConnector$ConnectorEndPoint) at org.mortbay.jetty.AbstractGenerator$Output.blockForOutput(AbstractGenerator.java:545) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:639) at org.mortbay.jetty.AbstractGenerator$Output.write(AbstractGenerator.java:580) at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:109) - locked <0x00007f7d116bbf90> (a org.mortbay.util.ByteArrayOutputStream2) at org.mortbay.jetty.AbstractGenerator$OutputWriter.write(AbstractGenerator.java:904) at java.io.Writer.write(Writer.java:96) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at java.io.PrintWriter.write(PrintWriter.java:361) - locked <0x00007f7d54b19ea0> (a org.mortbay.jetty.HttpConnection$OutputWriter) at org.jamon.escaping.HtmlEscaping.write(HtmlEscaping.java:43) at org.jamon.escaping.AbstractCharacterEscaping.write(AbstractCharacterEscaping.java:35) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmplImpl.renderNoFlush(RSStatusTmplImpl.java:222) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.renderNoFlush(RSStatusTmpl.java:180) at org.apache.hadoop.hbase.tmpl.regionserver.RSStatusTmpl.render(RSStatusTmpl.java:171) at org.apache.hadoop.hbase.regionserver.RSStatusServlet.doGet(RSStatusServlet.java:48) at javax.servlet.http.HttpServlet.service(HttpServlet.java:707) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) at org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:932) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "regionserver60020.decayingSampleTick.1" daemon prio=10 tid=0x0000000040b2b800 nid=0x6f1a waiting on condition [0x00007f7c697a5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb40cb0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) at java.util.concurrent.DelayQueue.take(DelayQueue.java:164) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) at java.lang.Thread.run(Thread.java:619) "LeaseRenewer:root@10.18.40.217:9000" daemon prio=10 tid=0x00007f7c6c770000 nid=0x6f18 waiting on condition [0x00007f7c699a7000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:432) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:69) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:299) at java.lang.Thread.run(Thread.java:619) "DestroyJavaVM" prio=10 tid=0x00007f7c6cb4a800 nid=0x6ee6 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "HftpFileSystem-DelegationTokenRenewer" daemon prio=10 tid=0x00007f7c6c648800 nid=0x6f12 waiting on condition [0x00007f7c69fad000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007f7c9eb0b648> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer.run(DelegationTokenRenewer.java:152) "Low Memory Detector" daemon prio=10 tid=0x000000004030c800 nid=0x6f02 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x00007f7c6c006800 nid=0x6f01 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x0000000040308000 nid=0x6f00 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Event Helper Thread" daemon prio=10 tid=0x00000000402f2800 nid=0x6eff runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "JDWP Transport Listener: dt_socket" daemon prio=10 tid=0x00000000402ef000 nid=0x6efe runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00000000402e1800 nid=0x6efd runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x00000000402df800 nid=0x6efc waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00000000402c3800 nid=0x6efb in Object.wait() [0x00007f7c71efe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00007f7c9eb77ac8> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x00000000402c1800 nid=0x6efa in Object.wait() [0x00007f7c71fff000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00007f7c9eb05110> (a java.lang.ref.Reference$Lock) "VM Thread" prio=10 tid=0x00000000402ba800 nid=0x6ef9 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x000000004011e800 nid=0x6ee7 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000040120800 nid=0x6ee8 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000040122800 nid=0x6ee9 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000040124000 nid=0x6eea runnable "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000040126000 nid=0x6eeb runnable "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000040128000 nid=0x6eec runnable "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000040129800 nid=0x6eed runnable "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x000000004012b800 nid=0x6eee runnable "Gang worker#8 (Parallel GC Threads)" prio=10 tid=0x000000004012d800 nid=0x6eef runnable "Gang worker#9 (Parallel GC Threads)" prio=10 tid=0x000000004012f000 nid=0x6ef0 runnable "Gang worker#10 (Parallel GC Threads)" prio=10 tid=0x0000000040131000 nid=0x6ef1 runnable "Gang worker#11 (Parallel GC Threads)" prio=10 tid=0x0000000040132800 nid=0x6ef2 runnable "Gang worker#12 (Parallel GC Threads)" prio=10 tid=0x0000000040134800 nid=0x6ef3 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x000000004026d000 nid=0x6ef8 runnable "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000040265800 nid=0x6ef4 runnable "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000040267800 nid=0x6ef5 runnable "Gang worker#2 (Parallel CMS Threads)" prio=10 tid=0x0000000040269800 nid=0x6ef6 runnable "Gang worker#3 (Parallel CMS Threads)" prio=10 tid=0x000000004026b000 nid=0x6ef7 runnable "VM Periodic Task Thread" prio=10 tid=0x0000000040317800 nid=0x6f03 waiting on condition JNI global references: 4197