Details
-
Bug
-
Status: Resolved
-
Major
-
Resolution: Fixed
-
None
-
None
Description
On running out of disk space whilst updating configuration store, thrown ServerScopedRuntimeException does not kill the broker.
Broker continues to run.
here is example of stack trace:
13:33:14.374 [VirtualHostNode-test-Config] ERROR o.a.q.s.u.ServerScopedRuntimeException - Cannot save to store java.io.IOException: No space left on device at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75] at java.io.FileOutputStream.write(FileOutputStream.java:345) ~[na:1.7.0_75] at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1846) ~[jackson-core-2.5.0.jar:2.5.0] at com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1070) ~[jackson-core-2.5.0.jar:2.5.0] at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.0.jar:2.5.0] at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2726) ~[jackson-databind-2.5.0.jar:2.5.0] at org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:412) ~[classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:408) ~[classes/:na] at org.apache.qpid.server.util.FileHelper.writeFileSafely(FileHelper.java:59) ~[classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:404) [classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572) [classes/:na] at org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143) [classes/:na] at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319) [guava-18.0.jar:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420) [classes/:na] at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106) [guava-18.0.jar:na] at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) [guava-18.0.jar:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361) [classes/:na] at java.security.AccessController.doPrivileged(Native Method) [na:1.7.0_75] at javax.security.auth.Subject.doAs(Subject.java:356) [na:1.7.0_75] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356) [classes/:na] at java.util.concurrent.FutureTask.run(FutureTask.java:262) [na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_75] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75] 13:33:14,374 |-ERROR in c.q.l.c.recovery.ResilientFileOutputStream@1367869763 - IO failure while writing to file [/mnt/vfs/work/log/qpid.log] java.io.IOException: No space left on device at java.io.IOException: No space left on device at at java.io.FileOutputStream.writeBytes(Native Method) at at java.io.FileOutputStream.write(FileOutputStream.java:345) at at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at at ch.qos.logback.core.recovery.ResilientOutputStreamBase.flush(ResilientOutputStreamBase.java:79) at at ch.qos.logback.core.encoder.LayoutWrappingEncoder.doEncode(LayoutWrappingEncoder.java:137) at at ch.qos.logback.core.OutputStreamAppender.writeOut(OutputStreamAppender.java:194) at at ch.qos.logback.core.FileAppender.writeOut(FileAppender.java:209) at at ch.qos.logback.core.OutputStreamAppender.subAppend(OutputStreamAppender.java:219) at at ch.qos.logback.core.rolling.RollingFileAppender.subAppend(RollingFileAppender.java:182) at at ch.qos.logback.core.OutputStreamAppender.append(OutputStreamAppender.java:103) at at ch.qos.logback.core.UnsynchronizedAppenderBase.doAppend(UnsynchronizedAppenderBase.java:88) at at ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:48) at at ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:273) at at ch.qos.logback.classic.Logger.callAppenders(Logger.java:260) at at ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:442) at at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:396) at at ch.qos.logback.classic.Logger.error(Logger.java:559) at at org.apache.qpid.server.util.ServerScopedRuntimeException.<init>(ServerScopedRuntimeException.java:40) at at org.apache.qpid.server.store.StoreException.<init>(StoreException.java:34) at at org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418) at at org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572) at at org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62) at at org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435) at at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729) at at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722) at at org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143) at at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319) at at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420) at at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106) at at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) at at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136) at at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085) at at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720) at at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715) at at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507) at at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500) at at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317) at at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48) at at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361) at at java.security.AccessController.doPrivileged(Native Method) at at javax.security.auth.Subject.doAs(Subject.java:356) at at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356) at at java.util.concurrent.FutureTask.run(FutureTask.java:262) at at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at at java.lang.Thread.run(Thread.java:745) 13:33:14.375 [VirtualHostNode-test-Config] ERROR o.a.q.s.logging.BrokerFileLoggerImpl - Unexpected I/O error whilst trying to write to log file. Log messages could be lost. java.io.IOException: No space left on device at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75] at java.io.FileOutputStream.write(FileOutputStream.java:345) ~[na:1.7.0_75] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[na:1.7.0_75] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[na:1.7.0_75] at ch.qos.logback.core.recovery.ResilientOutputStreamBase.flush(ResilientOutputStreamBase.java:79) ~[logback-core-1.1.2.jar:na] at ch.qos.logback.core.encoder.LayoutWrappingEncoder.doEncode(LayoutWrappingEncoder.java:137) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.OutputStreamAppender.writeOut(OutputStreamAppender.java:194) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.FileAppender.writeOut(FileAppender.java:209) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.OutputStreamAppender.subAppend(OutputStreamAppender.java:219) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.rolling.RollingFileAppender.subAppend(RollingFileAppender.java:182) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.OutputStreamAppender.append(OutputStreamAppender.java:103) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.UnsynchronizedAppenderBase.doAppend(UnsynchronizedAppenderBase.java:88) [logback-core-1.1.2.jar:na] at ch.qos.logback.core.spi.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:48) [logback-core-1.1.2.jar:na] at ch.qos.logback.classic.Logger.appendLoopOnAppenders(Logger.java:273) [logback-classic-1.1.2.jar:na] at ch.qos.logback.classic.Logger.callAppenders(Logger.java:260) [logback-classic-1.1.2.jar:na] at ch.qos.logback.classic.Logger.buildLoggingEventAndAppend(Logger.java:442) [logback-classic-1.1.2.jar:na] at ch.qos.logback.classic.Logger.filterAndLog_0_Or3Plus(Logger.java:396) [logback-classic-1.1.2.jar:na] at ch.qos.logback.classic.Logger.error(Logger.java:559) [logback-classic-1.1.2.jar:na] at org.apache.qpid.server.util.ServerScopedRuntimeException.<init>(ServerScopedRuntimeException.java:40) [classes/:na] at org.apache.qpid.server.store.StoreException.<init>(StoreException.java:34) [classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418) [classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572) [classes/:na] at org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143) [classes/:na] at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319) [guava-18.0.jar:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420) [classes/:na] at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106) [guava-18.0.jar:na] at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) [guava-18.0.jar:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507) [classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48) [classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361) [classes/:na] at java.security.AccessController.doPrivileged(Native Method) [na:1.7.0_75] at javax.security.auth.Subject.doAs(Subject.java:356) [na:1.7.0_75] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356) [classes/:na] at java.util.concurrent.FutureTask.run(FutureTask.java:262) [na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [na:1.7.0_75] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75] 13:33:14.377 [HttpManagement-58] WARN o.a.q.s.m.p.servlet.rest.RestServlet - Unexpected exception processing request org.apache.qpid.server.store.StoreException: Cannot save to store at org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:418) ~[classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.update(JsonFileConfigStore.java:572) ~[classes/:na] at org.apache.qpid.server.configuration.store.StoreConfigurationChangeListener.childAdded(StoreConfigurationChangeListener.java:62) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.childAdded(AbstractConfiguredObject.java:1435) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1729) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18$1.call(AbstractConfiguredObject.java:1722) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$21.onSuccess(AbstractConfiguredObject.java:2143) ~[classes/:na] at com.google.common.util.concurrent.Futures$6.run(Futures.java:1319) ~[guava-18.0.jar:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$ImmediateIfSameThreadExecutor.execute(TaskExecutorImpl.java:420) ~[classes/:na] at com.google.common.util.concurrent.Futures$ImmediateFuture.addListener(Futures.java:106) ~[guava-18.0.jar:na] at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1322) ~[guava-18.0.jar:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2136) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject.doAfter(AbstractConfiguredObject.java:2085) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1720) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$18.call(AbstractConfiguredObject.java:1715) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:507) ~[classes/:na] at org.apache.qpid.server.model.AbstractConfiguredObject$2.execute(AbstractConfiguredObject.java:500) ~[classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.executeTask(TaskExecutorImpl.java:317) ~[classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl.access$700(TaskExecutorImpl.java:48) ~[classes/:na] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper$1.run(TaskExecutorImpl.java:361) ~[classes/:na] at java.security.AccessController.doPrivileged(Native Method) ~[na:1.7.0_75] at javax.security.auth.Subject.doAs(Subject.java:356) ~[na:1.7.0_75] at org.apache.qpid.server.configuration.updater.TaskExecutorImpl$CallableWrapper.call(TaskExecutorImpl.java:356) ~[classes/:na] at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_75] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_75] at java.lang.Thread.run(Thread.java:745) [na:1.7.0_75] Caused by: java.io.IOException: No space left on device at java.io.FileOutputStream.writeBytes(Native Method) ~[na:1.7.0_75] at java.io.FileOutputStream.write(FileOutputStream.java:345) ~[na:1.7.0_75] at com.fasterxml.jackson.core.json.UTF8JsonGenerator._flushBuffer(UTF8JsonGenerator.java:1846) ~[jackson-core-2.5.0.jar:2.5.0] at com.fasterxml.jackson.core.json.UTF8JsonGenerator.close(UTF8JsonGenerator.java:1070) ~[jackson-core-2.5.0.jar:2.5.0] at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:3385) ~[jackson-databind-2.5.0.jar:2.5.0] at com.fasterxml.jackson.databind.ObjectMapper.writeValue(ObjectMapper.java:2726) ~[jackson-databind-2.5.0.jar:2.5.0] at org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:412) ~[classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore$3.performAction(JsonFileConfigStore.java:408) ~[classes/:na] at org.apache.qpid.server.util.FileHelper.writeFileSafely(FileHelper.java:59) ~[classes/:na] at org.apache.qpid.server.store.JsonFileConfigStore.save(JsonFileConfigStore.java:404) ~[classes/:na] ... 26 common frames omitted