********************* **** LOG APP **** ********************* 2017-03-05 03:45:06] org.apache.kafka.common.KafkaException: Record for partition <> at offset 449883 is invalid, cause: Record is corrupt (stored crc = 2171407101, computed crc = 1371274824) at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:749) at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:688) at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:424) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1045) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ********************* **** Trace JVM **** ********************* Full thread dump OpenJDK 64-Bit Server VM (25.121-b13 mixed mode): "Attach Listener" #67 daemon prio=9 os_prio=0 tid=0x00007f3450001000 nid=0xb3c2 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | acm_low_balance" #66 daemon prio=5 os_prio=0 tid=0x00007f33e433f800 nid=0x3e38 in Object.wait() [0x00007f34e1043000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:884) - locked <0x0000000701ffd118> (a org.apache.kafka.clients.consumer.internals.ConsumerCoordinator) Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | acm_low_balance" #65 daemon prio=5 os_prio=0 tid=0x00007f33d814f000 nid=0x3e37 in Object.wait() [0x00007f34e0f42000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:884) - locked <0x0000000701fe2f48> (a org.apache.kafka.clients.consumer.internals.ConsumerCoordinator) Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | acm_low_balance" #64 daemon prio=5 os_prio=0 tid=0x00007f33dc121000 nid=0x3e36 in Object.wait() [0x00007f34e0e41000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:884) - locked <0x0000000701fde770> (a org.apache.kafka.clients.consumer.internals.ConsumerCoordinator) Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | acm_low_balance" #63 daemon prio=5 os_prio=0 tid=0x00007f33e0299800 nid=0x3e35 in Object.wait() [0x00007f34e1144000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:884) - locked <0x0000000702000640> (a org.apache.kafka.clients.consumer.internals.ConsumerCoordinator) Locked ownable synchronizers: - None "kafka-coordinator-heartbeat-thread | acm_low_balance" #62 daemon prio=5 os_prio=0 tid=0x00007f33d018f000 nid=0x3e34 in Object.wait() [0x00007f34e0d40000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.kafka.clients.consumer.internals.AbstractCoordinator$HeartbeatThread.run(AbstractCoordinator.java:884) - locked <0x0000000701ffd328> (a org.apache.kafka.clients.consumer.internals.ConsumerCoordinator) Locked ownable synchronizers: - None "nof-consumer-acm_low_balance-4" #61 daemon prio=5 os_prio=0 tid=0x00007f33f020e800 nid=0x3e33 runnable [0x00007f34e195b000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.allocateMemory(Native Method) at java.nio.DirectByteBuffer.(DirectByteBuffer.java:127) at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) at com.esotericsoftware.kryo.io.ByteBufferInput.setBuffer(ByteBufferInput.java:96) at com.esotericsoftware.kryo.io.ByteBufferInput.(ByteBufferInput.java:64) TestKafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:57) TestKafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:21) at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:763) at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:688) at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:424) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1045) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) e() at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x0000000701ffe0e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nof-consumer-acm_low_balance-3" #59 daemon prio=5 os_prio=0 tid=0x00007f33f020d800 nid=0x3e32 runnable [0x00007f34e1444000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.allocateMemory(Native Method) at java.nio.DirectByteBuffer.(DirectByteBuffer.java:127) at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) at com.esotericsoftware.kryo.io.ByteBufferInput.setBuffer(ByteBufferInput.java:96) at com.esotericsoftware.kryo.io.ByteBufferInput.(ByteBufferInput.java:64) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:57) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:21) at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:763) at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:688) at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:424) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1045) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x0000000701ffd658> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nof-consumer-acm_low_balance-2" #57 daemon prio=5 os_prio=0 tid=0x00007f33f061f800 nid=0x3e31 runnable [0x00007f34e1646000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.allocateMemory(Native Method) at java.nio.DirectByteBuffer.(DirectByteBuffer.java:127) at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) at com.esotericsoftware.kryo.io.ByteBufferInput.setBuffer(ByteBufferInput.java:96) at com.esotericsoftware.kryo.io.ByteBufferInput.(ByteBufferInput.java:64) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:57) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:21) at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:763) at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:688) at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:424) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1045) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x0000000701fe6fc8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nof-consumer-acm_low_balance-1" #55 daemon prio=5 os_prio=0 tid=0x00007f33f0011000 nid=0x3e30 runnable [0x00007f34e1545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x0000000701feaad8> (a sun.nio.ch.Util$3) - locked <0x0000000701feaac8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000701fea2d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:470) at org.apache.kafka.common.network.Selector.poll(Selector.java:286) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260) at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:232) - locked <0x0000000702022148> (a org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1031) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x0000000702000900> (a java.util.concurrent.ThreadPoolExecutor$Worker) "nof-consumer-acm_low_balance-0" #53 daemon prio=5 os_prio=0 tid=0x00007f33f0010800 nid=0x3e2f runnable [0x00007f34e1a5c000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.allocateMemory(Native Method) at java.nio.DirectByteBuffer.(DirectByteBuffer.java:127) at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) at com.esotericsoftware.kryo.io.ByteBufferInput.setBuffer(ByteBufferInput.java:96) at com.esotericsoftware.kryo.io.ByteBufferInput.(ByteBufferInput.java:64) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:57) Test.KafkaKryoSerializer.deserialize(KafkaKryoSerializer.java:21) at org.apache.kafka.clients.consumer.internals.Fetcher.parseRecord(Fetcher.java:763) at org.apache.kafka.clients.consumer.internals.Fetcher.parseFetchedData(Fetcher.java:688) at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:424) at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:1045) at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:979) () at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:721) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157) at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:85) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629) at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618) at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:656) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - <0x0000000701fd6f58> (a java.util.concurrent.ThreadPoolExecutor$Worker) "logback-2" #51 daemon prio=5 os_prio=0 tid=0x00007f340c001000 nid=0x1d97 waiting on condition [0x00007f34e0c3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000070003e060> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Locked ownable synchronizers: - None "DestroyJavaVM" #50 prio=5 os_prio=0 tid=0x00007f350c009000 nid=0x13ee waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None