Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Duplicate
-
0.8.2
-
None
-
None
Description
Sometimes interpreter restart results in server deadlock, which can be caught with `jcmd <pid> Thread.print` with output like:
Found one Java-level deadlock: ============================= "qtp846238611-295773": waiting to lock monitor 0x00007f3cdc099408 (object 0x0000000673c66758, a java.util.LinkedList), which is held by "qtp846238611-295335" "qtp846238611-295335": waiting to lock monitor 0x00007f3e98023cd8 (object 0x000000065accda50, a java.util.ArrayList), which is held by "qtp846238611-292136" "qtp846238611-292136": waiting to lock monitor 0x00007f3cdc099408 (object 0x0000000673c66758, a java.util.LinkedList), which is held by "qtp846238611-295335"Java stack information for the threads listed above: =================================================== "qtp846238611-295773": at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750) - waiting to lock <0x0000000673c66758> (a java.util.LinkedList) at org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872) at org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568) at org.apache.zeppelin.socket.NotebookServer$ParagraphListenerImpl.afterStatusChange(NotebookServer.java:2333) at org.apache.zeppelin.scheduler.Job.setStatus(Job.java:162) at org.apache.zeppelin.scheduler.RemoteScheduler.submit(RemoteScheduler.java:154) at org.apache.zeppelin.notebook.Paragraph.execute(Paragraph.java:355) at org.apache.zeppelin.notebook.Note.run(Note.java:683) at org.apache.zeppelin.socket.NotebookServer.persistAndExecuteSingleParagraph(NotebookServer.java:1881) at org.apache.zeppelin.socket.NotebookServer.runParagraph(NotebookServer.java:1840) at org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:262) at org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189) at org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162) at org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459) at org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105) at org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85) at org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220) at org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219) at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) "qtp846238611-295335": at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750) - waiting to lock <0x000000065accda50> (a java.util.ArrayList) at org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872) at org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568) at org.apache.zeppelin.socket.NotebookServer$NotebookInformationListener.onParagraphRemove(NotebookServer.java:2194) at org.apache.zeppelin.notebook.Notebook.onParagraphRemove(Notebook.java:1084) at org.apache.zeppelin.notebook.Note.removeParagraph(Note.java:446) - locked <0x0000000673c66758> (a java.util.LinkedList) at org.apache.zeppelin.socket.NotebookServer.removeParagraph(NotebookServer.java:1363) at org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:283) at org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189) at org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162) at org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459) at org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105) at org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85) at org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220) at org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219) at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) "qtp846238611-292136": at org.apache.zeppelin.notebook.Note.getParagraphs(Note.java:750) - waiting to lock <0x0000000673c66758> (a java.util.LinkedList) at org.apache.zeppelin.notebook.Notebook.getJobListByUnixTime(Notebook.java:872) at org.apache.zeppelin.socket.NotebookServer.broadcastUpdateNoteJobInfo(NotebookServer.java:568) at org.apache.zeppelin.socket.NotebookServer$NotebookInformationListener.onParagraphRemove(NotebookServer.java:2194) at org.apache.zeppelin.notebook.Notebook.onParagraphRemove(Notebook.java:1084) at org.apache.zeppelin.notebook.Note.removeParagraph(Note.java:446) - locked <0x000000065accda50> (a java.util.ArrayList) at org.apache.zeppelin.socket.NotebookServer.removeParagraph(NotebookServer.java:1363) at org.apache.zeppelin.socket.NotebookServer.onMessage(NotebookServer.java:283) at org.apache.zeppelin.socket.NotebookSocket.onWebSocketText(NotebookSocket.java:59) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextMessage(JettyListenerEventDriver.java:189) at org.eclipse.jetty.websocket.common.message.SimpleTextMessage.messageComplete(SimpleTextMessage.java:69) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.appendMessage(AbstractEventDriver.java:66) at org.eclipse.jetty.websocket.common.events.JettyListenerEventDriver.onTextFrame(JettyListenerEventDriver.java:158) at org.eclipse.jetty.websocket.common.events.AbstractEventDriver.incomingFrame(AbstractEventDriver.java:162) at org.eclipse.jetty.websocket.common.WebSocketSession.incomingFrame(WebSocketSession.java:459) at org.eclipse.jetty.websocket.common.extensions.AbstractExtension.nextIncomingFrame(AbstractExtension.java:182) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.nextIncomingFrame(PerMessageDeflateExtension.java:105) at org.eclipse.jetty.websocket.common.extensions.compress.CompressExtension.forwardIncoming(CompressExtension.java:142) at org.eclipse.jetty.websocket.common.extensions.compress.PerMessageDeflateExtension.incomingFrame(PerMessageDeflateExtension.java:85) at org.eclipse.jetty.websocket.common.extensions.ExtensionStack.incomingFrame(ExtensionStack.java:220) at org.eclipse.jetty.websocket.common.Parser.notifyFrame(Parser.java:219) at org.eclipse.jetty.websocket.common.Parser.parse(Parser.java:244) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.readParse(AbstractWebSocketConnection.java:559) at org.eclipse.jetty.websocket.common.io.AbstractWebSocketConnection.onFillable(AbstractWebSocketConnection.java:390) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)