Details
Critical Description
The nimbus was restarted during HA testing. After the restart the nimbus failed to come up.
2017-01-18 04:57:58.231 o.a.s.s.o.a.c.f.s.ConnectionStateManager [INFO] State change: CONNECTED
2017-01-18 04:57:58.247 o.a.s.b.BlobStoreUtils [ERROR] Could not update the blob with keyKillLeaderThenSubmitNewTopology1-1-1484715309-stormjar.jar
2017-01-18 04:57:58.273 o.a.s.b.KeySequenceNumber [ERROR] Exception {}
org.apache.storm.shade.org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /blobstore/KillLeaderThenSubmitNewTopology1-1-1484715309-stormjar.jar
at org.apache.storm.shade.org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
at org.apache.storm.shade.org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.storm.shade.org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1590)
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:214)
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:203)
at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:108)
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:200)
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:191)
at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:38)
at org.apache.storm.blobstore.KeySequenceNumber.getKeySequenceNumber(KeySequenceNumber.java:149)
at org.apache.storm.daemon.nimbus$get_version_for_key.invoke(nimbus.clj:456)
at org.apache.storm.daemon.nimbus$mk_reified_nimbus$reify__9548.createStateInZookeeper(nimbus.clj:2056)
at org.apache.storm.generated.Nimbus$Processor$createStateInZookeeper.getResult(Nimbus.java:3755)
at org.apache.storm.generated.Nimbus$Processor$createStateInZookeeper.getResult(Nimbus.java:3740)
at org.apache.storm.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.storm.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.storm.security.auth.SaslTransportPlugin$TUGIWrapProcessor.process(SaslTransportPlugin.java:144)
at org.apache.storm.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2017-01-18 04:57:58.274 o.a.s.s.o.a.c.f.i.CuratorFrameworkImpl [INFO] backgroundOperationsLoop exiting
2017-01-18 04:57:58.296 o.a.s.m.n.Login [INFO] successfully logged in.
2017-01-18 04:57:58.309 o.a.s.s.o.a.z.ZooKeeper [INFO] Session: 0x359afc1eaa2009b closed
2017-01-18 04:57:58.309 o.a.s.s.o.a.z.ClientCnxn [INFO] EventThread shut down
2017-01-18 04:57:58.310 o.a.s.t.s.TThreadPoolServer [ERROR] Error occurred during processing of message.
java.util.NoSuchElementException
at java.util.TreeMap.key(TreeMap.java:1327)
at java.util.TreeMap.lastKey(TreeMap.java:297)
at java.util.TreeSet.last(TreeSet.java:401)
at org.apache.storm.blobstore.KeySequenceNumber.getKeySequenceNumber(KeySequenceNumber.java:206)
at org.apache.storm.daemon.nimbus$get_version_for_key.invoke(nimbus.clj:456)
at org.apache.storm.daemon.nimbus$mk_reified_nimbus$reify__9548.createStateInZookeeper(nimbus.clj:2056)
at org.apache.storm.generated.Nimbus$Processor$createStateInZookeeper.getResult(Nimbus.java:3755)
at org.apache.storm.generated.Nimbus$Processor$createStateInZookeeper.getResult(Nimbus.java:3740)
at org.apache.storm.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.storm.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.storm.security.auth.SaslTransportPlugin$TUGIWrapProcessor.process(SaslTransportPlugin.java:144)
at org.apache.storm.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:286)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
2017-01-18 04:57:58.311 o.a.s.d.nimbus [ERROR] Error when processing event
java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException
at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:92)
at org.apache.storm.daemon.nimbus$fn__9373.invoke(nimbus.clj:1452)
at clojure.lang.MultiFn.invoke(MultiFn.java:233)
at org.apache.storm.daemon.nimbus$fn__9770$exec_fn__3656__auto____9771$fn__9786.invoke(nimbus.clj:2452)
at org.apache.storm.timer$schedule_recurring$this__2188.invoke(timer.clj:105)
at org.apache.storm.timer$mk_timer$fn__2171$fn__2172.invoke(timer.clj:50)
at org.apache.storm.timer$mk_timer$fn__2171.invoke(timer.clj:42)
at clojure.lang.AFn.run(AFn.java:22)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException
at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:114)
at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:76)
... 8 more
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException
at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:252)
at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:111)
... 9 more
Caused by: java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException
at org.apache.storm.blobstore.NimbusBlobStore.createStateInZookeeper(NimbusBlobStore.java:349)
at org.apache.storm.blobstore.BlobStoreUtils.createStateInZookeeper(BlobStoreUtils.java:217)
at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:249)
... 10 more
Caused by: org.apache.storm.thrift.transport.TTransportException
at org.apache.storm.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132)
at org.apache.storm.thrift.transport.TTransport.readAll(TTransport.java:86)
at org.apache.storm.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:376)
at org.apache.storm.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:453)
at org.apache.storm.thrift.transport.TSaslTransport.read(TSaslTransport.java:435)
at org.apache.storm.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37)
at org.apache.storm.thrift.transport.TTransport.readAll(TTransport.java:86)
at org.apache.storm.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429)
at org.apache.storm.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318)
at org.apache.storm.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219)
at org.apache.storm.thrift.TServiceClient.receiveBase(TServiceClient.java:77)
at org.apache.storm.generated.Nimbus$Client.recv_createStateInZookeeper(Nimbus.java:1000)
at org.apache.storm.generated.Nimbus$Client.createStateInZookeeper(Nimbus.java:987)
at org.apache.storm.blobstore.NimbusBlobStore.createStateInZookeeper(NimbusBlobStore.java:346)
... 12 more
2017-01-18 04:57:58.314 o.a.s.util [ERROR] Halting process: ("Error when processing an event")
java.lang.RuntimeException: ("Error when processing an event")
at org.apache.storm.util$exit_process_BANG_.doInvoke(util.clj:341)
at clojure.lang.RestFn.invoke(RestFn.java:423)
at org.apache.storm.daemon.nimbus$nimbus_data$fn__8579.invoke(nimbus.clj:212)
at org.apache.storm.timer$mk_timer$fn__2171$fn__2172.invoke(timer.clj:71)
at org.apache.storm.timer$mk_timer$fn__2171.invoke(timer.clj:42)
at clojure.lang.AFn.run(AFn.java:22)
at java.lang.Thread.run(Thread.java:745)
2017-01-18 04:57:58,317 FATAL Ignoring log event after log4j was shut down
2017-01-18 04:57:58,317 FATAL Ignoring log event after log4j was shut down
2017-01-18 04:57:58,317 FATAL Ignoring log event after log4j was shut down
2017-01-18 04:57:58,318 FATAL Ignoring log event after log4j was shut down
2017-01-18 04:57:58,318 FATAL Ignoring log event after log4j was shut down
