Index: core/src/test/scala/unit/kafka/message/ByteBufferMessageSetTest.scala =================================================================== --- core/src/test/scala/unit/kafka/message/ByteBufferMessageSetTest.scala (revision 1185769) +++ core/src/test/scala/unit/kafka/message/ByteBufferMessageSetTest.scala (working copy) @@ -21,6 +21,7 @@ import junit.framework.Assert._ import org.junit.Test import kafka.utils.TestUtils +import kafka.common.InvalidMessageSizeException class ByteBufferMessageSetTest extends BaseMessageSetTestCases { @@ -28,6 +29,24 @@ new ByteBufferMessageSet(NoCompressionCodec, messages: _*) @Test + def testSmallFetchSize() { + // create a ByteBufferMessageSet that doesn't contain a full message + // iterating it should get an InvalidMessageSizeException + val messages = new ByteBufferMessageSet(NoCompressionCodec, new Message("01234567890123456789".getBytes())) + val buffer = messages.serialized.slice + buffer.limit(10) + val messageSetWithNoFullMessage = new ByteBufferMessageSet(buffer = buffer, initialOffset = 1000) + try { + for (message <- messageSetWithNoFullMessage) + fail("shouldn't see any message") + } + catch { + case e: InvalidMessageSizeException => //this is expected + case e2 => fail("shouldn't see any other exceptions") + } + } + + @Test def testValidBytes() { { val messages = new ByteBufferMessageSet(NoCompressionCodec, new Message("hello".getBytes()), new Message("there".getBytes())) Index: core/src/main/scala/kafka/tools/ConsumerShell.scala =================================================================== --- core/src/main/scala/kafka/tools/ConsumerShell.scala (revision 1185769) +++ core/src/main/scala/kafka/tools/ConsumerShell.scala (working copy) @@ -98,7 +98,7 @@ } }catch { case e:ConsumerTimeoutException => // this is ok - case oe: Exception => logger.error(oe) + case oe: Exception => logger.error("error in ZKConsumerThread", oe) } shutdownLatch.countDown println("Received " + count + " messages") Index: core/src/main/scala/kafka/message/ByteBufferMessageSet.scala =================================================================== --- core/src/main/scala/kafka/message/ByteBufferMessageSet.scala (revision 1185769) +++ core/src/main/scala/kafka/message/ByteBufferMessageSet.scala (working copy) @@ -99,7 +99,7 @@ logger.trace("size of data = " + size) } if(size < 0 || topIter.remaining < size) { - if (currValidBytes == 0 || size < 0) + if (currValidBytes == initialOffset || size < 0) throw new InvalidMessageSizeException("invalid message size: " + size + " only received bytes: " + topIter.remaining + " at " + currValidBytes + "( possible causes (1) a single message larger than " + "the fetch size; (2) log corruption )") Index: core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala =================================================================== --- core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala (revision 1185769) +++ core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala (working copy) @@ -76,7 +76,8 @@ * add an empty message with the exception to the queue so that client can see the error */ def enqueueError(e: Throwable, fetchOffset: Long) = { - val messages = new ByteBufferMessageSet(ErrorMapping.EmptyByteBuffer, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) + val messages = new ByteBufferMessageSet(buffer = ErrorMapping.EmptyByteBuffer, initialOffset = 0, + errorCode = ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) chunkQueue.put(new FetchedDataChunk(messages, this, fetchOffset)) }