diff --git a/bin/kafka-producer-shell.sh b/bin/kafka-producer-shell.sh deleted file mode 100755 index 3f75a34..0000000 --- a/bin/kafka-producer-shell.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -$(dirname $0)/kafka-run-class.sh kafka.tools.ProducerShell $@ diff --git a/bin/windows/kafka-server-stop.bat b/bin/windows/kafka-server-stop.bat new file mode 100644 index 0000000..b9496ce --- /dev/null +++ b/bin/windows/kafka-server-stop.bat @@ -0,0 +1,18 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +wmic process where (commandline like "%%kafka.Kafka%%" and not name="wmic.exe") delete +rem ps ax | grep -i 'kafka.Kafka' | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM diff --git a/bin/windows/zookeeper-server-stop.bat b/bin/windows/zookeeper-server-stop.bat new file mode 100644 index 0000000..29bdee8 --- /dev/null +++ b/bin/windows/zookeeper-server-stop.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +wmic process where (commandline like "%%zookeeper%%" and not name="wmic.exe") delete diff --git a/contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java b/contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java index b0e75bc..8e98efc 100644 --- a/contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java +++ b/contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java @@ -109,7 +109,7 @@ public class KafkaETLContext { // read data from queue URI uri = _request.getURI(); - _consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), _timeout, _bufferSize); + _consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), _timeout, _bufferSize, "KafkaETLContext"); // get available offset range _offsetRange = getOffsetRange(); diff --git a/core/src/main/scala/kafka/admin/AdminUtils.scala b/core/src/main/scala/kafka/admin/AdminUtils.scala index 40976c5..437a685 100644 --- a/core/src/main/scala/kafka/admin/AdminUtils.scala +++ b/core/src/main/scala/kafka/admin/AdminUtils.scala @@ -25,7 +25,8 @@ import org.I0Itec.zkclient.ZkClient import org.I0Itec.zkclient.exception.ZkNodeExistsException import scala.collection._ import scala.collection.mutable -import kafka.common.{BrokerNotAvailableException, LeaderNotAvailableException, ReplicaNotAvailableException, ErrorMapping} +import kafka.common._ +import scala.Some object AdminUtils extends Logging { val rand = new Random @@ -82,7 +83,7 @@ object AdminUtils extends Logging { ZkUtils.createPersistentPath(zkClient, zkPath, jsonPartitionMap) debug("Updated path %s with %s for replica assignment".format(zkPath, jsonPartitionMap)) } catch { - case e: ZkNodeExistsException => throw new AdministrationException("topic %s already exists".format(topic)) + case e: ZkNodeExistsException => throw new TopicExistsException("topic %s already exists".format(topic)) case e2 => throw new AdministrationException(e2.toString) } } diff --git a/core/src/main/scala/kafka/api/FetchRequest.scala b/core/src/main/scala/kafka/api/FetchRequest.scala index 9892fb3..b4fb874 100644 --- a/core/src/main/scala/kafka/api/FetchRequest.scala +++ b/core/src/main/scala/kafka/api/FetchRequest.scala @@ -30,10 +30,10 @@ case class PartitionFetchInfo(offset: Long, fetchSize: Int) object FetchRequest { - val CurrentVersion = 1.shortValue() + val CurrentVersion = 0.shortValue val DefaultMaxWait = 0 val DefaultMinBytes = 0 - val ReplicaFetcherClientId = "replica fetcher" + val ReplicaFetcherClientId = "replica-fetcher" val DefaultCorrelationId = 0 def readFrom(buffer: ByteBuffer): FetchRequest = { diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 24253c7..94650f1 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -27,29 +27,25 @@ import kafka.api.ApiUtils._ object FetchResponsePartitionData { def readFrom(buffer: ByteBuffer): FetchResponsePartitionData = { val error = buffer.getShort - val initialOffset = buffer.getLong val hw = buffer.getLong val messageSetSize = buffer.getInt val messageSetBuffer = buffer.slice() messageSetBuffer.limit(messageSetSize) buffer.position(buffer.position + messageSetSize) - new FetchResponsePartitionData(error, initialOffset, - hw, new ByteBufferMessageSet(messageSetBuffer)) + new FetchResponsePartitionData(error, hw, new ByteBufferMessageSet(messageSetBuffer)) } val headerSize = 2 + /* error code */ - 8 + /* initialOffset */ 8 + /* high watermark */ 4 /* messageSetSize */ } -case class FetchResponsePartitionData(error: Short = ErrorMapping.NoError, - initialOffset:Long = 0L, hw: Long = -1L, messages: MessageSet) { +case class FetchResponsePartitionData(error: Short = ErrorMapping.NoError, hw: Long = -1L, messages: MessageSet) { val sizeInBytes = FetchResponsePartitionData.headerSize + messages.sizeInBytes - def this(messages: MessageSet) = this(ErrorMapping.NoError, 0L, -1L, messages) + def this(messages: MessageSet) = this(ErrorMapping.NoError, -1L, messages) } @@ -63,7 +59,6 @@ class PartitionDataSend(val partitionId: Int, private val buffer = ByteBuffer.allocate( 4 /** partitionId **/ + FetchResponsePartitionData.headerSize) buffer.putInt(partitionId) buffer.putShort(partitionData.error) - buffer.putLong(partitionData.initialOffset) buffer.putLong(partitionData.hw) buffer.putInt(partitionData.messages.sizeInBytes) buffer.rewind() @@ -141,12 +136,10 @@ class TopicDataSend(val topicData: TopicData) extends Send { object FetchResponse { val headerSize = - 2 + /* versionId */ 4 + /* correlationId */ 4 /* topic count */ def readFrom(buffer: ByteBuffer): FetchResponse = { - val versionId = buffer.getShort val correlationId = buffer.getInt val topicCount = buffer.getInt val pairs = (1 to topicCount).flatMap(_ => { @@ -156,13 +149,12 @@ object FetchResponse { (TopicAndPartition(topicData.topic, partitionId), partitionData) } }) - FetchResponse(versionId, correlationId, Map(pairs:_*)) + FetchResponse(correlationId, Map(pairs:_*)) } } -case class FetchResponse(versionId: Short, - correlationId: Int, +case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchResponsePartitionData]) { /** @@ -211,7 +203,6 @@ class FetchResponseSend(val fetchResponse: FetchResponse) extends Send { private val buffer = ByteBuffer.allocate(4 /* for size */ + FetchResponse.headerSize) buffer.putInt(size) - buffer.putShort(fetchResponse.versionId) buffer.putInt(fetchResponse.correlationId) buffer.putInt(fetchResponse.dataGroupedByTopic.size) // topic count buffer.rewind() diff --git a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala index f57de6e..9759949 100644 --- a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala +++ b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala @@ -79,7 +79,7 @@ case class PartitionStateInfo(val leaderIsrAndControllerEpoch: LeaderIsrAndContr } object LeaderAndIsrRequest { - val CurrentVersion = 1.shortValue() + val CurrentVersion = 0.shortValue val DefaultClientId = "" val IsInit: Boolean = true val NotInit: Boolean = false @@ -87,6 +87,7 @@ object LeaderAndIsrRequest { def readFrom(buffer: ByteBuffer): LeaderAndIsrRequest = { val versionId = buffer.getShort + val correlationId = buffer.getInt val clientId = readShortString(buffer) val ackTimeoutMs = buffer.getInt val controllerEpoch = buffer.getInt @@ -106,11 +107,12 @@ object LeaderAndIsrRequest { for (i <- 0 until leadersCount) leaders += Broker.readFrom(buffer) - new LeaderAndIsrRequest(versionId, clientId, ackTimeoutMs, partitionStateInfos.toMap, leaders, controllerEpoch) + new LeaderAndIsrRequest(versionId, correlationId, clientId, ackTimeoutMs, partitionStateInfos.toMap, leaders, controllerEpoch) } } case class LeaderAndIsrRequest (versionId: Short, + correlationId: Int, clientId: String, ackTimeoutMs: Int, partitionStateInfos: Map[(String, Int), PartitionStateInfo], @@ -119,12 +121,13 @@ case class LeaderAndIsrRequest (versionId: Short, extends RequestOrResponse(Some(RequestKeys.LeaderAndIsrKey)) { def this(partitionStateInfos: Map[(String, Int), PartitionStateInfo], liveBrokers: Set[Broker], controllerEpoch: Int) = { - this(LeaderAndIsrRequest.CurrentVersion, LeaderAndIsrRequest.DefaultClientId, LeaderAndIsrRequest.DefaultAckTimeout, + this(LeaderAndIsrRequest.CurrentVersion, 0, LeaderAndIsrRequest.DefaultClientId, LeaderAndIsrRequest.DefaultAckTimeout, partitionStateInfos, liveBrokers, controllerEpoch) } def writeTo(buffer: ByteBuffer) { buffer.putShort(versionId) + buffer.putInt(correlationId) writeShortString(buffer, clientId) buffer.putInt(ackTimeoutMs) buffer.putInt(controllerEpoch) @@ -141,6 +144,7 @@ case class LeaderAndIsrRequest (versionId: Short, def sizeInBytes(): Int = { var size = 2 /* version id */ + + 4 /* correlation id */ + (2 + clientId.length) /* client id */ + 4 /* ack timeout */ + 4 /* controller epoch */ + diff --git a/core/src/main/scala/kafka/api/LeaderAndIsrResponse.scala b/core/src/main/scala/kafka/api/LeaderAndIsrResponse.scala index f2e86be..dbd85d0 100644 --- a/core/src/main/scala/kafka/api/LeaderAndIsrResponse.scala +++ b/core/src/main/scala/kafka/api/LeaderAndIsrResponse.scala @@ -26,7 +26,7 @@ import collection.Map object LeaderAndIsrResponse { def readFrom(buffer: ByteBuffer): LeaderAndIsrResponse = { - val versionId = buffer.getShort + val correlationId = buffer.getInt val errorCode = buffer.getShort val numEntries = buffer.getInt val responseMap = new HashMap[(String, Int), Short]() @@ -36,18 +36,18 @@ object LeaderAndIsrResponse { val partitionErrorCode = buffer.getShort responseMap.put((topic, partition), partitionErrorCode) } - new LeaderAndIsrResponse(versionId, responseMap, errorCode) + new LeaderAndIsrResponse(correlationId, responseMap, errorCode) } } -case class LeaderAndIsrResponse(versionId: Short, +case class LeaderAndIsrResponse(correlationId: Int, responseMap: Map[(String, Int), Short], errorCode: Short = ErrorMapping.NoError) extends RequestOrResponse { def sizeInBytes(): Int ={ var size = - 2 /* version id */ + + 4 /* correlation id */ + 2 /* error code */ + 4 /* number of responses */ for ((key, value) <- responseMap) { @@ -60,7 +60,7 @@ case class LeaderAndIsrResponse(versionId: Short, } def writeTo(buffer: ByteBuffer) { - buffer.putShort(versionId) + buffer.putInt(correlationId) buffer.putShort(errorCode) buffer.putInt(responseMap.size) for ((key:(String, Int), value) <- responseMap){ diff --git a/core/src/main/scala/kafka/api/OffsetRequest.scala b/core/src/main/scala/kafka/api/OffsetRequest.scala index ee3dff5..6c522bc 100644 --- a/core/src/main/scala/kafka/api/OffsetRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetRequest.scala @@ -23,7 +23,7 @@ import kafka.api.ApiUtils._ object OffsetRequest { - val CurrentVersion = 1.shortValue() + val CurrentVersion = 0.shortValue val DefaultClientId = "" val SmallestTimeString = "smallest" @@ -33,6 +33,7 @@ object OffsetRequest { def readFrom(buffer: ByteBuffer): OffsetRequest = { val versionId = buffer.getShort + val correlationId = buffer.getInt val clientId = readShortString(buffer) val replicaId = buffer.getInt val topicCount = buffer.getInt @@ -54,16 +55,18 @@ case class PartitionOffsetRequestInfo(time: Long, maxNumOffsets: Int) case class OffsetRequest(requestInfo: Map[TopicAndPartition, PartitionOffsetRequestInfo], versionId: Short = OffsetRequest.CurrentVersion, + correlationId: Int = 0, clientId: String = OffsetRequest.DefaultClientId, replicaId: Int = Request.OrdinaryConsumerId) extends RequestOrResponse(Some(RequestKeys.OffsetsKey)) { - def this(requestInfo: Map[TopicAndPartition, PartitionOffsetRequestInfo], replicaId: Int) = this(requestInfo, OffsetRequest.CurrentVersion, OffsetRequest.DefaultClientId, replicaId) + def this(requestInfo: Map[TopicAndPartition, PartitionOffsetRequestInfo], correlationId: Int, replicaId: Int) = this(requestInfo, OffsetRequest.CurrentVersion, correlationId, OffsetRequest.DefaultClientId, replicaId) lazy val requestInfoGroupedByTopic = requestInfo.groupBy(_._1.topic) def writeTo(buffer: ByteBuffer) { buffer.putShort(versionId) + buffer.putInt(correlationId) writeShortString(buffer, clientId) buffer.putInt(replicaId) @@ -83,6 +86,7 @@ case class OffsetRequest(requestInfo: Map[TopicAndPartition, PartitionOffsetRequ def sizeInBytes = 2 + /* versionId */ + 4 + /* correlationId */ shortStringLength(clientId) + 4 + /* replicaId */ 4 + /* topic count */ diff --git a/core/src/main/scala/kafka/api/OffsetResponse.scala b/core/src/main/scala/kafka/api/OffsetResponse.scala index 10a7715..264e200 100644 --- a/core/src/main/scala/kafka/api/OffsetResponse.scala +++ b/core/src/main/scala/kafka/api/OffsetResponse.scala @@ -25,7 +25,7 @@ import kafka.api.ApiUtils._ object OffsetResponse { def readFrom(buffer: ByteBuffer): OffsetResponse = { - val versionId = buffer.getShort + val correlationId = buffer.getInt val numTopics = buffer.getInt val pairs = (1 to numTopics).flatMap(_ => { val topic = readShortString(buffer) @@ -38,7 +38,7 @@ object OffsetResponse { (TopicAndPartition(topic, partition), PartitionOffsetsResponse(error, offsets)) }) }) - OffsetResponse(versionId, Map(pairs:_*)) + OffsetResponse(correlationId, Map(pairs:_*)) } } @@ -47,7 +47,7 @@ object OffsetResponse { case class PartitionOffsetsResponse(error: Short, offsets: Seq[Long]) -case class OffsetResponse(versionId: Short, +case class OffsetResponse(correlationId: Int, partitionErrorAndOffsets: Map[TopicAndPartition, PartitionOffsetsResponse]) extends RequestOrResponse { @@ -56,7 +56,7 @@ case class OffsetResponse(versionId: Short, def hasError = partitionErrorAndOffsets.values.exists(_.error != ErrorMapping.NoError) val sizeInBytes = { - 2 + /* versionId */ + 4 + /* correlation id */ 4 + /* topic count */ offsetsGroupedByTopic.foldLeft(0)((foldedTopics, currTopic) => { val (topic, errorAndOffsetsMap) = currTopic @@ -74,7 +74,7 @@ case class OffsetResponse(versionId: Short, } def writeTo(buffer: ByteBuffer) { - buffer.putShort(versionId) + buffer.putInt(correlationId) buffer.putInt(offsetsGroupedByTopic.size) // topic count offsetsGroupedByTopic.foreach { case((topic, errorAndOffsetsMap)) => diff --git a/core/src/main/scala/kafka/api/ProducerRequest.scala b/core/src/main/scala/kafka/api/ProducerRequest.scala index 1713dee..9edc4dd 100644 --- a/core/src/main/scala/kafka/api/ProducerRequest.scala +++ b/core/src/main/scala/kafka/api/ProducerRequest.scala @@ -25,7 +25,7 @@ import kafka.api.ApiUtils._ object ProducerRequest { - val CurrentVersion: Short = 0 + val CurrentVersion = 0.shortValue def readFrom(buffer: ByteBuffer): ProducerRequest = { val versionId: Short = buffer.getShort @@ -57,7 +57,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, clientId: String, requiredAcks: Short, ackTimeoutMs: Int, - data: Map[TopicAndPartition, MessageSet]) + data: Map[TopicAndPartition, ByteBufferMessageSet]) extends RequestOrResponse(Some(RequestKeys.ProduceKey)) { /** @@ -69,7 +69,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, clientId: String, requiredAcks: Short, ackTimeoutMs: Int, - data: Map[TopicAndPartition, MessageSet]) = + data: Map[TopicAndPartition, ByteBufferMessageSet]) = this(ProducerRequest.CurrentVersion, correlationId, clientId, requiredAcks, ackTimeoutMs, data) def writeTo(buffer: ByteBuffer) { @@ -88,7 +88,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, topicAndPartitionData.foreach(partitionAndData => { val partition = partitionAndData._1.partition val partitionMessageData = partitionAndData._2 - val bytes = partitionMessageData.asInstanceOf[ByteBufferMessageSet].buffer + val bytes = partitionMessageData.buffer buffer.putInt(partition) buffer.putInt(bytes.limit) buffer.put(bytes) diff --git a/core/src/main/scala/kafka/api/ProducerResponse.scala b/core/src/main/scala/kafka/api/ProducerResponse.scala index 62c9bc4..743227d 100644 --- a/core/src/main/scala/kafka/api/ProducerResponse.scala +++ b/core/src/main/scala/kafka/api/ProducerResponse.scala @@ -25,7 +25,6 @@ import kafka.api.ApiUtils._ object ProducerResponse { def readFrom(buffer: ByteBuffer): ProducerResponse = { - val versionId = buffer.getShort val correlationId = buffer.getInt val topicCount = buffer.getInt val statusPairs = (1 to topicCount).flatMap(_ => { @@ -39,15 +38,14 @@ object ProducerResponse { }) }) - ProducerResponse(versionId, correlationId, Map(statusPairs:_*)) + ProducerResponse(correlationId, Map(statusPairs:_*)) } } case class ProducerResponseStatus(error: Short, offset: Long) -case class ProducerResponse(versionId: Short, - correlationId: Int, +case class ProducerResponse(correlationId: Int, status: Map[TopicAndPartition, ProducerResponseStatus]) extends RequestOrResponse { /** @@ -59,7 +57,6 @@ case class ProducerResponse(versionId: Short, val sizeInBytes = { val groupedStatus = statusGroupedByTopic - 2 + /* version id */ 4 + /* correlation id */ 4 + /* topic count */ groupedStatus.foldLeft (0) ((foldedTopics, currTopic) => { @@ -76,8 +73,6 @@ case class ProducerResponse(versionId: Short, def writeTo(buffer: ByteBuffer) { val groupedStatus = statusGroupedByTopic - - buffer.putShort(versionId) buffer.putInt(correlationId) buffer.putInt(groupedStatus.size) // topic count diff --git a/core/src/main/scala/kafka/api/StopReplicaRequest.scala b/core/src/main/scala/kafka/api/StopReplicaRequest.scala index 9088fa9..deb195f 100644 --- a/core/src/main/scala/kafka/api/StopReplicaRequest.scala +++ b/core/src/main/scala/kafka/api/StopReplicaRequest.scala @@ -25,12 +25,13 @@ import kafka.network.InvalidRequestException object StopReplicaRequest extends Logging { - val CurrentVersion = 1.shortValue() + val CurrentVersion = 0.shortValue val DefaultClientId = "" val DefaultAckTimeout = 100 def readFrom(buffer: ByteBuffer): StopReplicaRequest = { val versionId = buffer.getShort + val correlationId = buffer.getInt val clientId = readShortString(buffer) val ackTimeoutMs = buffer.getInt val controllerEpoch = buffer.getInt @@ -45,11 +46,12 @@ object StopReplicaRequest extends Logging { (1 to topicPartitionPairCount) foreach { _ => topicPartitionPairSet.add(readShortString(buffer), buffer.getInt) } - StopReplicaRequest(versionId, clientId, ackTimeoutMs, deletePartitions, topicPartitionPairSet.toSet, controllerEpoch) + StopReplicaRequest(versionId, correlationId, clientId, ackTimeoutMs, deletePartitions, topicPartitionPairSet.toSet, controllerEpoch) } } case class StopReplicaRequest(versionId: Short, + correlationId: Int, clientId: String, ackTimeoutMs: Int, deletePartitions: Boolean, @@ -58,12 +60,13 @@ case class StopReplicaRequest(versionId: Short, extends RequestOrResponse(Some(RequestKeys.StopReplicaKey)) { def this(deletePartitions: Boolean, partitions: Set[(String, Int)], controllerEpoch: Int) = { - this(StopReplicaRequest.CurrentVersion, StopReplicaRequest.DefaultClientId, StopReplicaRequest.DefaultAckTimeout, + this(StopReplicaRequest.CurrentVersion, 0, StopReplicaRequest.DefaultClientId, StopReplicaRequest.DefaultAckTimeout, deletePartitions, partitions, controllerEpoch) } def writeTo(buffer: ByteBuffer) { buffer.putShort(versionId) + buffer.putInt(correlationId) writeShortString(buffer, clientId) buffer.putInt(ackTimeoutMs) buffer.putInt(controllerEpoch) @@ -78,6 +81,7 @@ case class StopReplicaRequest(versionId: Short, def sizeInBytes(): Int = { var size = 2 + /* versionId */ + 4 + /* correlation id */ ApiUtils.shortStringLength(clientId) + 4 + /* ackTimeoutMs */ 4 + /* controller epoch */ diff --git a/core/src/main/scala/kafka/api/StopReplicaResponse.scala b/core/src/main/scala/kafka/api/StopReplicaResponse.scala index e0d3de6..fa66b99 100644 --- a/core/src/main/scala/kafka/api/StopReplicaResponse.scala +++ b/core/src/main/scala/kafka/api/StopReplicaResponse.scala @@ -26,7 +26,7 @@ import kafka.api.ApiUtils._ object StopReplicaResponse { def readFrom(buffer: ByteBuffer): StopReplicaResponse = { - val versionId = buffer.getShort + val correlationId = buffer.getInt val errorCode = buffer.getShort val numEntries = buffer.getInt @@ -37,17 +37,17 @@ object StopReplicaResponse { val partitionErrorCode = buffer.getShort() responseMap.put((topic, partition), partitionErrorCode) } - new StopReplicaResponse(versionId, responseMap.toMap, errorCode) + new StopReplicaResponse(correlationId, responseMap.toMap, errorCode) } } -case class StopReplicaResponse(val versionId: Short, +case class StopReplicaResponse(val correlationId: Int, val responseMap: Map[(String, Int), Short], val errorCode: Short = ErrorMapping.NoError) extends RequestOrResponse{ def sizeInBytes(): Int ={ var size = - 2 /* version id */ + + 4 /* correlation id */ + 2 /* error code */ + 4 /* number of responses */ for ((key, value) <- responseMap) { @@ -60,7 +60,7 @@ case class StopReplicaResponse(val versionId: Short, } def writeTo(buffer: ByteBuffer) { - buffer.putShort(versionId) + buffer.putInt(correlationId) buffer.putShort(errorCode) buffer.putInt(responseMap.size) for ((key:(String, Int), value) <- responseMap){ diff --git a/core/src/main/scala/kafka/api/TopicMetadata.scala b/core/src/main/scala/kafka/api/TopicMetadata.scala index e2d03e8..409de76 100644 --- a/core/src/main/scala/kafka/api/TopicMetadata.scala +++ b/core/src/main/scala/kafka/api/TopicMetadata.scala @@ -21,57 +21,29 @@ import kafka.cluster.Broker import java.nio.ByteBuffer import kafka.api.ApiUtils._ import kafka.utils.Logging -import collection.mutable.ListBuffer -import kafka.common.{KafkaException, ErrorMapping} - -/** - * topic (2 bytes + topic.length) - * number of partitions (4 bytes) - * - * partition id (4 bytes) - * - * does leader exist (1 byte) - * leader info (4 + creator.length + host.length + 4 (port) + 4 (id)) - * number of replicas (2 bytes) - * replica info (4 + creator.length + host.length + 4 (port) + 4 (id)) - * number of in sync replicas (2 bytes) - * replica info (4 + creator.length + host.length + 4 (port) + 4 (id)) - * - * does log metadata exist (1 byte) - * number of log segments (4 bytes) - * total size of log in bytes (8 bytes) - * - * number of log segments (4 bytes) - * beginning offset (8 bytes) - * last modified timestamp (8 bytes) - * size of log segment (8 bytes) - * - */ - -sealed trait LeaderRequest { def requestId: Byte } -case object LeaderExists extends LeaderRequest { val requestId: Byte = 1 } -case object LeaderDoesNotExist extends LeaderRequest { val requestId: Byte = 0 } +import collection.mutable.ArrayBuffer +import kafka.common._ object TopicMetadata { + + val NoLeaderNodeId = -1 - def readFrom(buffer: ByteBuffer): TopicMetadata = { + def readFrom(buffer: ByteBuffer, brokers: Map[Int, Broker]): TopicMetadata = { val errorCode = readShortInRange(buffer, "error code", (-1, Short.MaxValue)) val topic = readShortString(buffer) val numPartitions = readIntInRange(buffer, "number of partitions", (0, Int.MaxValue)) - val partitionsMetadata = new ListBuffer[PartitionMetadata]() + val partitionsMetadata = new ArrayBuffer[PartitionMetadata]() for(i <- 0 until numPartitions) - partitionsMetadata += PartitionMetadata.readFrom(buffer) + partitionsMetadata += PartitionMetadata.readFrom(buffer, brokers) new TopicMetadata(topic, partitionsMetadata, errorCode) } } case class TopicMetadata(topic: String, partitionsMetadata: Seq[PartitionMetadata], errorCode: Short = ErrorMapping.NoError) extends Logging { def sizeInBytes: Int = { - var size: Int = 2 /* error code */ - size += shortStringLength(topic) - size += partitionsMetadata.foldLeft(4 /* number of partitions */)(_ + _.sizeInBytes) - debug("Size of topic metadata = " + size) - size + 2 /* error code */ + + shortStringLength(topic) + + 4 + partitionsMetadata.map(_.sizeInBytes).sum /* size and partition data array */ } def writeTo(buffer: ByteBuffer) { @@ -87,40 +59,24 @@ case class TopicMetadata(topic: String, partitionsMetadata: Seq[PartitionMetadat object PartitionMetadata { - def readFrom(buffer: ByteBuffer): PartitionMetadata = { + def readFrom(buffer: ByteBuffer, brokers: Map[Int, Broker]): PartitionMetadata = { val errorCode = readShortInRange(buffer, "error code", (-1, Short.MaxValue)) val partitionId = readIntInRange(buffer, "partition id", (0, Int.MaxValue)) /* partition id */ - val doesLeaderExist = getLeaderRequest(buffer.get) - val leader = doesLeaderExist match { - case LeaderExists => /* leader exists */ - Some(Broker.readFrom(buffer)) - case LeaderDoesNotExist => None - } + val leaderId = buffer.getInt + val leader = brokers.get(leaderId) /* list of all replicas */ - val numReplicas = readShortInRange(buffer, "number of all replicas", (0, Short.MaxValue)) - val replicas = new Array[Broker](numReplicas) - for(i <- 0 until numReplicas) { - replicas(i) = Broker.readFrom(buffer) - } + val numReplicas = readIntInRange(buffer, "number of all replicas", (0, Int.MaxValue)) + val replicaIds = (0 until numReplicas).map(_ => buffer.getInt) + val replicas = replicaIds.map(brokers) /* list of in-sync replicas */ - val numIsr = readShortInRange(buffer, "number of in-sync replicas", (0, Short.MaxValue)) - val isr = new Array[Broker](numIsr) - for(i <- 0 until numIsr) { - isr(i) = Broker.readFrom(buffer) - } + val numIsr = readIntInRange(buffer, "number of in-sync replicas", (0, Int.MaxValue)) + val isrIds = (0 until numIsr).map(_ => buffer.getInt) + val isr = isrIds.map(brokers) new PartitionMetadata(partitionId, leader, replicas, isr, errorCode) } - - private def getLeaderRequest(requestId: Byte): LeaderRequest = { - requestId match { - case LeaderExists.requestId => LeaderExists - case LeaderDoesNotExist.requestId => LeaderDoesNotExist - case _ => throw new KafkaException("Unknown leader request id " + requestId) - } - } } case class PartitionMetadata(partitionId: Int, @@ -129,42 +85,28 @@ case class PartitionMetadata(partitionId: Int, isr: Seq[Broker] = Seq.empty, errorCode: Short = ErrorMapping.NoError) extends Logging { def sizeInBytes: Int = { - var size: Int = 2 /* error code */ + 4 /* partition id */ + 1 /* if leader exists*/ - - leader match { - case Some(l) => size += l.sizeInBytes - case None => - } - - size += 2 /* number of replicas */ - size += replicas.foldLeft(0)(_ + _.sizeInBytes) - size += 2 /* number of in sync replicas */ - size += isr.foldLeft(0)(_ + _.sizeInBytes) - - debug("Size of partition metadata = " + size) - size + 2 /* error code */ + + 4 /* partition id */ + + 4 /* leader */ + + 4 + 4 * replicas.size /* replica array */ + + 4 + 4 * isr.size /* isr array */ } def writeTo(buffer: ByteBuffer) { buffer.putShort(errorCode) buffer.putInt(partitionId) - /* if leader exists*/ - leader match { - case Some(l) => - buffer.put(LeaderExists.requestId) - /* leader id host_name port */ - l.writeTo(buffer) - case None => buffer.put(LeaderDoesNotExist.requestId) - } + /* leader */ + val leaderId = if(leader.isDefined) leader.get.id else TopicMetadata.NoLeaderNodeId + buffer.putInt(leaderId) /* number of replicas */ - buffer.putShort(replicas.size.toShort) - replicas.foreach(r => r.writeTo(buffer)) + buffer.putInt(replicas.size) + replicas.foreach(r => buffer.putInt(r.id)) /* number of in-sync replicas */ - buffer.putShort(isr.size.toShort) - isr.foreach(r => r.writeTo(buffer)) + buffer.putInt(isr.size) + isr.foreach(r => buffer.putInt(r.id)) } } diff --git a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala index 70c42e3..5bdb2c1 100644 --- a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala @@ -23,7 +23,7 @@ import collection.mutable.ListBuffer import kafka.utils.Logging object TopicMetadataRequest extends Logging { - val CurrentVersion = 1.shortValue() + val CurrentVersion = 0.shortValue val DefaultClientId = "" /** @@ -33,6 +33,7 @@ object TopicMetadataRequest extends Logging { def readFrom(buffer: ByteBuffer): TopicMetadataRequest = { val versionId = buffer.getShort + val correlationId = buffer.getInt val clientId = readShortString(buffer) val numTopics = readIntInRange(buffer, "number of topics", (0, Int.MaxValue)) val topics = new ListBuffer[String]() @@ -40,26 +41,28 @@ object TopicMetadataRequest extends Logging { topics += readShortString(buffer) val topicsList = topics.toList debug("topic = %s".format(topicsList.head)) - new TopicMetadataRequest(versionId, clientId, topics.toList) + new TopicMetadataRequest(versionId, clientId, topics.toList, correlationId) } } case class TopicMetadataRequest(val versionId: Short, val clientId: String, - val topics: Seq[String]) + val topics: Seq[String], + val correlationId: Int) extends RequestOrResponse(Some(RequestKeys.MetadataKey)){ def this(topics: Seq[String]) = - this(TopicMetadataRequest.CurrentVersion, TopicMetadataRequest.DefaultClientId, topics) + this(TopicMetadataRequest.CurrentVersion, TopicMetadataRequest.DefaultClientId, topics, 0) def writeTo(buffer: ByteBuffer) { buffer.putShort(versionId) + buffer.putInt(correlationId) // correlation id not set yet writeShortString(buffer, clientId) buffer.putInt(topics.size) topics.foreach(topic => writeShortString(buffer, topic)) } def sizeInBytes(): Int = { - 2 + (2 + clientId.length) + 4 /* number of topics */ + topics.foldLeft(0)(_ + shortStringLength(_)) /* topics */ + 2 + 4 + shortStringLength(clientId) + 4 /* number of topics */ + topics.foldLeft(0)(_ + shortStringLength(_)) /* topics */ } } diff --git a/core/src/main/scala/kafka/api/TopicMetadataResponse.scala b/core/src/main/scala/kafka/api/TopicMetadataResponse.scala index 25068d1..af76776 100644 --- a/core/src/main/scala/kafka/api/TopicMetadataResponse.scala +++ b/core/src/main/scala/kafka/api/TopicMetadataResponse.scala @@ -17,30 +17,43 @@ package kafka.api +import kafka.cluster.Broker import java.nio.ByteBuffer object TopicMetadataResponse { def readFrom(buffer: ByteBuffer): TopicMetadataResponse = { - val versionId = buffer.getShort + val correlationId = buffer.getInt + val brokerCount = buffer.getInt + val brokers = (0 until brokerCount).map(_ => Broker.readFrom(buffer)) + val brokerMap = brokers.map(b => (b.id, b)).toMap val topicCount = buffer.getInt - val topicsMetadata = new Array[TopicMetadata](topicCount) - for( i <- 0 until topicCount) { - topicsMetadata(i) = TopicMetadata.readFrom(buffer) - } - new TopicMetadataResponse(versionId, topicsMetadata.toSeq) + val topicsMetadata = (0 until topicCount).map(_ => TopicMetadata.readFrom(buffer, brokerMap)) + new TopicMetadataResponse(topicsMetadata, correlationId) } } -case class TopicMetadataResponse(versionId: Short, - topicsMetadata: Seq[TopicMetadata]) extends RequestOrResponse -{ - val sizeInBytes = 2 + topicsMetadata.foldLeft(4)(_ + _.sizeInBytes) +case class TopicMetadataResponse(topicsMetadata: Seq[TopicMetadata], + correlationId: Int) extends RequestOrResponse { + val sizeInBytes: Int = { + val brokers = extractBrokers(topicsMetadata).values + 4 + 4 + brokers.map(_.sizeInBytes).sum + 4 + topicsMetadata.map(_.sizeInBytes).sum + } def writeTo(buffer: ByteBuffer) { - buffer.putShort(versionId) + buffer.putInt(correlationId) + /* brokers */ + val brokers = extractBrokers(topicsMetadata).values + buffer.putInt(brokers.size) + brokers.foreach(_.writeTo(buffer)) /* topic metadata */ buffer.putInt(topicsMetadata.length) topicsMetadata.foreach(_.writeTo(buffer)) } + + def extractBrokers(topicMetadatas: Seq[TopicMetadata]): Map[Int, Broker] = { + val parts = topicsMetadata.flatMap(_.partitionsMetadata) + val brokers = (parts.flatMap(_.replicas)) ++ (parts.map(_.leader).collect{case Some(l) => l}) + brokers.map(b => (b.id, b)).toMap + } } diff --git a/core/src/main/scala/kafka/client/ClientUtils.scala b/core/src/main/scala/kafka/client/ClientUtils.scala index aeead2d..c61833b 100644 --- a/core/src/main/scala/kafka/client/ClientUtils.scala +++ b/core/src/main/scala/kafka/client/ClientUtils.scala @@ -6,20 +6,28 @@ import kafka.api._ import kafka.producer._ import kafka.common.KafkaException import kafka.utils.{Utils, Logging} +import java.util.Properties /** * Helper functions common to clients (producer, consumer, or admin) */ object ClientUtils extends Logging{ - def fetchTopicMetadata(topics: Set[String], brokers: Seq[Broker]): TopicMetadataResponse = { + /** + * Used by the producer to send a metadata request since it has access to the ProducerConfig + * @param topics The topics for which the metadata needs to be fetched + * @param brokers The brokers in the cluster as configured on the producer through broker.list + * @param producerConfig The producer's config + * @return topic metadata response + */ + def fetchTopicMetadata(topics: Set[String], brokers: Seq[Broker], producerConfig: ProducerConfig): TopicMetadataResponse = { var fetchMetaDataSucceeded: Boolean = false var i: Int = 0 val topicMetadataRequest = new TopicMetadataRequest(topics.toSeq) var topicMetadataResponse: TopicMetadataResponse = null var t: Throwable = null while(i < brokers.size && !fetchMetaDataSucceeded) { - val producer: SyncProducer = ProducerPool.createSyncProducer(None, brokers(i)) + val producer: SyncProducer = ProducerPool.createSyncProducer(producerConfig, brokers(i)) info("Fetching metadata for topic %s".format(topics)) try { topicMetadataResponse = producer.send(topicMetadataRequest) @@ -39,7 +47,22 @@ object ClientUtils extends Logging{ } return topicMetadataResponse } - + + /** + * Used by a non-producer client to send a metadata request + * @param topics The topics for which the metadata needs to be fetched + * @param brokers The brokers in the cluster as configured on the client + * @param clientId The client's identifier + * @return topic metadata response + */ + def fetchTopicMetadata(topics: Set[String], brokers: Seq[Broker], clientId: String): TopicMetadataResponse = { + val props = new Properties() + props.put("broker.list", brokers.map(_.getConnectionString()).mkString(",")) + props.put("clientid", clientId) + val producerConfig = new ProducerConfig(props) + fetchTopicMetadata(topics, brokers, producerConfig) + } + /** * Parse a list of broker urls in the form host1:port1, host2:port2, ... */ @@ -52,8 +75,7 @@ object ClientUtils extends Logging{ val brokerInfos = brokerStr.split(":") val hostName = brokerInfos(0) val port = brokerInfos(1).toInt - val creatorId = hostName + "-" + System.currentTimeMillis() - new Broker(brokerId, creatorId, hostName, port) + new Broker(brokerId, hostName, port) }) } diff --git a/core/src/main/scala/kafka/cluster/Broker.scala b/core/src/main/scala/kafka/cluster/Broker.scala index 03a75f0..ffedecd 100644 --- a/core/src/main/scala/kafka/cluster/Broker.scala +++ b/core/src/main/scala/kafka/cluster/Broker.scala @@ -31,36 +31,32 @@ private[kafka] object Broker { if(brokerInfoString == null) throw new BrokerNotAvailableException("Broker id %s does not exist".format(id)) val brokerInfo = brokerInfoString.split(":") - new Broker(id, brokerInfo(0), brokerInfo(1), brokerInfo(2).toInt) + new Broker(id, brokerInfo(0), brokerInfo(1).toInt) } def readFrom(buffer: ByteBuffer): Broker = { val id = buffer.getInt - val creatorId = readShortString(buffer) val host = readShortString(buffer) val port = buffer.getInt - new Broker(id, creatorId, host, port) + new Broker(id, host, port) } } -private[kafka] case class Broker(val id: Int, val creatorId: String, val host: String, val port: Int) { +private[kafka] case class Broker(val id: Int, val host: String, val port: Int) { - override def toString(): String = new String("id:" + id + ",creatorId:" + creatorId + ",host:" + host + ",port:" + port) + override def toString(): String = new String("id:" + id + ",host:" + host + ",port:" + port) - def getZKString(): String = new String(creatorId + ":" + host + ":" + port) + def getZkString(): String = host + ":" + port + + def getConnectionString(): String = host + ":" + port def writeTo(buffer: ByteBuffer) { buffer.putInt(id) - writeShortString(buffer, creatorId) writeShortString(buffer, host) buffer.putInt(port) } - def sizeInBytes: Int = { - val size = shortStringLength(creatorId) + shortStringLength(host) /* host name */ + 4 /* port */ + 4 /* broker id*/ - debug("Size of broker info = " + size) - size - } + def sizeInBytes: Int = shortStringLength(host) /* host name */ + 4 /* port */ + 4 /* broker id*/ override def equals(obj: Any): Boolean = { obj match { diff --git a/core/src/main/scala/kafka/common/InvalidClientIdException.scala b/core/src/main/scala/kafka/common/InvalidClientIdException.scala new file mode 100644 index 0000000..edf072d --- /dev/null +++ b/core/src/main/scala/kafka/common/InvalidClientIdException.scala @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.common + +class InvalidClientIdException(message: String) extends RuntimeException(message) { + def this() = this(null) +} diff --git a/core/src/main/scala/kafka/common/TopicExistsException.scala b/core/src/main/scala/kafka/common/TopicExistsException.scala new file mode 100644 index 0000000..88a084e --- /dev/null +++ b/core/src/main/scala/kafka/common/TopicExistsException.scala @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.common + +class TopicExistsException(message: String) extends RuntimeException(message) { + def this() = this(null) +} \ No newline at end of file diff --git a/core/src/main/scala/kafka/consumer/ConsoleConsumer.scala b/core/src/main/scala/kafka/consumer/ConsoleConsumer.scala index 2a8e02d..9378357 100644 --- a/core/src/main/scala/kafka/consumer/ConsoleConsumer.scala +++ b/core/src/main/scala/kafka/consumer/ConsoleConsumer.scala @@ -89,7 +89,7 @@ object ConsoleConsumer extends Logging { .withRequiredArg .describedAs("class") .ofType(classOf[String]) - .defaultsTo(classOf[NewlineMessageFormatter].getName) + .defaultsTo(classOf[DefaultMessageFormatter].getName) val messageFormatterArgOpt = parser.accepts("property") .withRequiredArg .describedAs("prop") @@ -176,6 +176,7 @@ object ConsoleConsumer extends Logging { } }) + var numMessages = 0L val formatter: MessageFormatter = messageFormatterClass.newInstance().asInstanceOf[MessageFormatter] formatter.init(formatterArgs) try { @@ -188,6 +189,7 @@ object ConsoleConsumer extends Logging { for(messageAndTopic <- iter) { try { formatter.writeTo(messageAndTopic.key, messageAndTopic.message, System.out) + numMessages += 1 } catch { case e => if (skipMessageOnError) @@ -198,6 +200,7 @@ object ConsoleConsumer extends Logging { if(System.out.checkError()) { // This means no one is listening to our output stream any more, time to shutdown System.err.println("Unable to write to standard out, closing consumer.") + System.err.println("Consumed %d messages".format(numMessages)) formatter.close() connector.shutdown() System.exit(1) @@ -206,6 +209,7 @@ object ConsoleConsumer extends Logging { } catch { case e => error("Error processing message, stopping consumer: ", e) } + System.out.println("Consumed %d messages".format(numMessages)) System.out.flush() formatter.close() connector.shutdown() @@ -256,10 +260,27 @@ trait MessageFormatter { def close() {} } -class NewlineMessageFormatter extends MessageFormatter { +class DefaultMessageFormatter extends MessageFormatter { + var printKey = false + var keySeparator = "\t".getBytes + var lineSeparator = "\n".getBytes + + override def init(props: Properties) { + if(props.containsKey("print.key")) + printKey = props.getProperty("print.key").trim.toLowerCase.equals("true") + if(props.containsKey("key.separator")) + keySeparator = props.getProperty("key.separator").getBytes + if(props.containsKey("line.separator")) + lineSeparator = props.getProperty("line.separator").getBytes + } + def writeTo(key: Array[Byte], value: Array[Byte], output: PrintStream) { + if(printKey) { + output.write(key) + output.write(keySeparator) + } output.write(value) - output.write('\n') + output.write(lineSeparator) } } diff --git a/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala b/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala index bdc020b..a6cbfb6 100644 --- a/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala +++ b/core/src/main/scala/kafka/consumer/ConsumerFetcherManager.scala @@ -54,7 +54,7 @@ class ConsumerFetcherManager(private val consumerIdString: String, try { trace("Partitions without leader %s".format(noLeaderPartitionSet)) val brokers = getAllBrokersInCluster(zkClient) - val topicsMetadata = ClientUtils.fetchTopicMetadata(noLeaderPartitionSet.map(m => m.topic).toSet, brokers).topicsMetadata + val topicsMetadata = ClientUtils.fetchTopicMetadata(noLeaderPartitionSet.map(m => m.topic).toSet, brokers, config.clientId).topicsMetadata val leaderForPartitionsMap = new HashMap[TopicAndPartition, Broker] topicsMetadata.foreach( tmd => { diff --git a/core/src/main/scala/kafka/consumer/ConsumerIterator.scala b/core/src/main/scala/kafka/consumer/ConsumerIterator.scala index fd80104..c5062fc 100644 --- a/core/src/main/scala/kafka/consumer/ConsumerIterator.scala +++ b/core/src/main/scala/kafka/consumer/ConsumerIterator.scala @@ -34,7 +34,8 @@ class ConsumerIterator[K, V](private val channel: BlockingQueue[FetchedDataChunk consumerTimeoutMs: Int, private val keyDecoder: Decoder[K], private val valueDecoder: Decoder[V], - val enableShallowIterator: Boolean) + val enableShallowIterator: Boolean, + val consumerTopicStats: ConsumerTopicStats) extends IteratorTemplate[MessageAndMetadata[K, V]] with Logging { private var current: AtomicReference[Iterator[MessageAndOffset]] = new AtomicReference(null) @@ -48,8 +49,8 @@ class ConsumerIterator[K, V](private val channel: BlockingQueue[FetchedDataChunk currentTopicInfo.resetConsumeOffset(consumedOffset) val topic = currentTopicInfo.topic trace("Setting %s consumed offset to %d".format(topic, consumedOffset)) - ConsumerTopicStat.getConsumerTopicStat(topic).messageRate.mark() - ConsumerTopicStat.getConsumerAllTopicStat().messageRate.mark() + consumerTopicStats.getConsumerTopicStats(topic).messageRate.mark() + consumerTopicStats.getConsumerAllTopicStats().messageRate.mark() item } diff --git a/core/src/main/scala/kafka/consumer/ConsumerTopicStat.scala b/core/src/main/scala/kafka/consumer/ConsumerTopicStat.scala index 2c9c204..e69de29 100644 --- a/core/src/main/scala/kafka/consumer/ConsumerTopicStat.scala +++ b/core/src/main/scala/kafka/consumer/ConsumerTopicStat.scala @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.consumer - -import kafka.utils.{Pool, threadsafe, Logging} -import java.util.concurrent.TimeUnit -import kafka.metrics.KafkaMetricsGroup - -@threadsafe -class ConsumerTopicStat(name: String) extends KafkaMetricsGroup { - val messageRate = newMeter(name + "MessagesPerSec", "messages", TimeUnit.SECONDS) - val byteRate = newMeter(name + "BytesPerSec", "bytes", TimeUnit.SECONDS) -} - -object ConsumerTopicStat extends Logging { - private val valueFactory = (k: String) => new ConsumerTopicStat(k) - private val stats = new Pool[String, ConsumerTopicStat](Some(valueFactory)) - private val allTopicStat = new ConsumerTopicStat("AllTopics") - - def getConsumerAllTopicStat(): ConsumerTopicStat = allTopicStat - - def getConsumerTopicStat(topic: String): ConsumerTopicStat = { - stats.getAndMaybePut(topic + "-") - } -} diff --git a/core/src/main/scala/kafka/consumer/ConsumerTopicStats.scala b/core/src/main/scala/kafka/consumer/ConsumerTopicStats.scala new file mode 100644 index 0000000..2a9d9fb --- /dev/null +++ b/core/src/main/scala/kafka/consumer/ConsumerTopicStats.scala @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.consumer + +import kafka.utils.{ClientIdAndTopic, Pool, threadsafe, Logging} +import java.util.concurrent.TimeUnit +import kafka.metrics.KafkaMetricsGroup + +@threadsafe +class ConsumerTopicMetrics(clientIdTopic: ClientIdAndTopic) extends KafkaMetricsGroup { + val messageRate = newMeter(clientIdTopic + "-MessagesPerSec", "messages", TimeUnit.SECONDS) + val byteRate = newMeter(clientIdTopic + "-BytesPerSec", "bytes", TimeUnit.SECONDS) +} + +class ConsumerTopicStats(clientId: String) extends Logging { + private val valueFactory = (k: ClientIdAndTopic) => new ConsumerTopicMetrics(k) + private val stats = new Pool[ClientIdAndTopic, ConsumerTopicMetrics](Some(valueFactory)) + private val allTopicStats = new ConsumerTopicMetrics(new ClientIdAndTopic(clientId, "AllTopics")) + + def getConsumerAllTopicStats(): ConsumerTopicMetrics = allTopicStats + + def getConsumerTopicStats(topic: String): ConsumerTopicMetrics = { + stats.getAndMaybePut(new ClientIdAndTopic(clientId, topic)) + } +} + diff --git a/core/src/main/scala/kafka/consumer/KafkaStream.scala b/core/src/main/scala/kafka/consumer/KafkaStream.scala index 115d41a..569f6df 100644 --- a/core/src/main/scala/kafka/consumer/KafkaStream.scala +++ b/core/src/main/scala/kafka/consumer/KafkaStream.scala @@ -26,11 +26,12 @@ class KafkaStream[K,V](private val queue: BlockingQueue[FetchedDataChunk], consumerTimeoutMs: Int, private val keyDecoder: Decoder[K], private val valueDecoder: Decoder[V], - val enableShallowIterator: Boolean) + val enableShallowIterator: Boolean, + val consumerTopicStats: ConsumerTopicStats) extends Iterable[MessageAndMetadata[K,V]] with java.lang.Iterable[MessageAndMetadata[K,V]] { private val iter: ConsumerIterator[K,V] = - new ConsumerIterator[K,V](queue, consumerTimeoutMs, keyDecoder, valueDecoder, enableShallowIterator) + new ConsumerIterator[K,V](queue, consumerTimeoutMs, keyDecoder, valueDecoder, enableShallowIterator, consumerTopicStats) /** * Create an iterator over messages in the stream. diff --git a/core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala b/core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala index 5249ddc..8c42d11 100644 --- a/core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala +++ b/core/src/main/scala/kafka/consumer/PartitionTopicInfo.scala @@ -28,7 +28,8 @@ class PartitionTopicInfo(val topic: String, private val chunkQueue: BlockingQueue[FetchedDataChunk], private val consumedOffset: AtomicLong, private val fetchedOffset: AtomicLong, - private val fetchSize: AtomicInteger) extends Logging { + private val fetchSize: AtomicInteger, + private val consumerTopicStats: ConsumerTopicStats) extends Logging { debug("initial consumer offset of " + this + " is " + consumedOffset.get) debug("initial fetch offset of " + this + " is " + fetchedOffset.get) @@ -58,8 +59,8 @@ class PartitionTopicInfo(val topic: String, chunkQueue.put(new FetchedDataChunk(messages, this, fetchedOffset.get)) fetchedOffset.set(next) debug("updated fetch offset of (%s) to %d".format(this, next)) - ConsumerTopicStat.getConsumerTopicStat(topic).byteRate.mark(size) - ConsumerTopicStat.getConsumerAllTopicStat().byteRate.mark(size) + consumerTopicStats.getConsumerTopicStats(topic).byteRate.mark(size) + consumerTopicStats.getConsumerAllTopicStats().byteRate.mark(size) } } diff --git a/core/src/main/scala/kafka/consumer/SimpleConsumer.scala b/core/src/main/scala/kafka/consumer/SimpleConsumer.scala index d642a67..5e1e6ab 100644 --- a/core/src/main/scala/kafka/consumer/SimpleConsumer.scala +++ b/core/src/main/scala/kafka/consumer/SimpleConsumer.scala @@ -30,19 +30,23 @@ import kafka.cluster.Broker object SimpleConsumer extends Logging { - def earliestOrLatestOffset(broker: Broker, topic: String, partitionId: Int, earliestOrLatest: Long, + def earliestOrLatestOffset(broker: Broker, + topic: String, + partitionId: Int, + earliestOrLatest: Long, + clientId: String, isFromOrdinaryConsumer: Boolean): Long = { var simpleConsumer: SimpleConsumer = null var producedOffset: Long = -1L try { simpleConsumer = new SimpleConsumer(broker.host, broker.port, ConsumerConfig.SocketTimeout, - ConsumerConfig.SocketBufferSize) + ConsumerConfig.SocketBufferSize, clientId) val topicAndPartition = TopicAndPartition(topic, partitionId) val request = if(isFromOrdinaryConsumer) new OffsetRequest(immutable.Map(topicAndPartition -> PartitionOffsetRequestInfo(earliestOrLatest, 1))) else new OffsetRequest(immutable.Map(topicAndPartition -> PartitionOffsetRequestInfo(earliestOrLatest, 1)), - Request.DebuggingConsumerId) + 0, Request.DebuggingConsumerId) producedOffset = simpleConsumer.getOffsetsBefore(request).partitionErrorAndOffsets(topicAndPartition).offsets.head } catch { case e => @@ -55,15 +59,20 @@ object SimpleConsumer extends Logging { producedOffset } - def earliestOrLatestOffset(zkClient: ZkClient, topic: String, brokerId: Int, partitionId: Int, - earliestOrLatest: Long, isFromOrdinaryConsumer: Boolean = true): Long = { + def earliestOrLatestOffset(zkClient: ZkClient, + topic: String, + brokerId: Int, + partitionId: Int, + earliestOrLatest: Long, + clientId: String, + isFromOrdinaryConsumer: Boolean = true): Long = { val cluster = getCluster(zkClient) val broker = cluster.getBroker(brokerId) match { case Some(b) => b case None => throw new KafkaException("Broker " + brokerId + " is unavailable. Cannot issue " + "getOffsetsBefore request") } - earliestOrLatestOffset(broker, topic, partitionId, earliestOrLatest, isFromOrdinaryConsumer) + earliestOrLatestOffset(broker, topic, partitionId, earliestOrLatest, clientId, isFromOrdinaryConsumer) } } @@ -75,10 +84,13 @@ object SimpleConsumer extends Logging { class SimpleConsumer(val host: String, val port: Int, val soTimeout: Int, - val bufferSize: Int) extends Logging { + val bufferSize: Int, + val clientId: String) extends Logging { + ClientId.validate(clientId) private val lock = new Object() private val blockingChannel = new BlockingChannel(host, port, bufferSize, BlockingChannel.UseDefaultBufferSize, soTimeout) + private val fetchRequestAndResponseStats = new FetchRequestAndResponseStats(clientId + "-host_%s-port_%s".format(host, port)) private def connect(): BlockingChannel = { close @@ -143,12 +155,12 @@ class SimpleConsumer(val host: String, */ def fetch(request: FetchRequest): FetchResponse = { var response: Receive = null - FetchRequestAndResponseStat.requestTimer.time { + fetchRequestAndResponseStats.requestTimer.time { response = sendRequest(request) } val fetchResponse = FetchResponse.readFrom(response.buffer) val fetchedSize = fetchResponse.sizeInBytes - FetchRequestAndResponseStat.respondSizeHist.update(fetchedSize) + fetchRequestAndResponseStats.respondSizeHist.update(fetchedSize) fetchResponse } @@ -166,7 +178,7 @@ class SimpleConsumer(val host: String, } } -object FetchRequestAndResponseStat extends KafkaMetricsGroup { - val requestTimer = new KafkaTimer(newTimer("FetchRequestRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS)) - val respondSizeHist = newHistogram("FetchResponseSize") +class FetchRequestAndResponseStats(clientId: String) extends KafkaMetricsGroup { + val requestTimer = new KafkaTimer(newTimer(clientId + "-FetchRequestRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS)) + val respondSizeHist = newHistogram(clientId + "-FetchResponseSize") } diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 79ecb8c..a9b0fd9 100644 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -35,7 +35,6 @@ import kafka.client.ClientUtils import com.yammer.metrics.core.Gauge import kafka.api.OffsetRequest import kafka.metrics._ -import kafka.producer.ProducerConfig /** @@ -80,6 +79,8 @@ private[kafka] object ZookeeperConsumerConnector { private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val enableFetcher: Boolean) // for testing only extends ConsumerConnector with Logging with KafkaMetricsGroup { + + ClientId.validate(config.clientId) private val isShuttingDown = new AtomicBoolean(false) private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None @@ -94,6 +95,8 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private var wildcardTopicWatcher: ZookeeperTopicEventWatcher = null + private val consumerTopicStats = new ConsumerTopicStats(config.clientId) + val consumerIdString = { var consumerUuid : String = null config.consumerId match { @@ -198,7 +201,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, threadIdSet.map(_ => { val queue = new LinkedBlockingQueue[FetchedDataChunk](config.maxQueuedChunks) val stream = new KafkaStream[K,V]( - queue, config.consumerTimeoutMs, keyDecoder, valueDecoder, config.enableShallowIterator) + queue, config.consumerTimeoutMs, keyDecoder, valueDecoder, config.enableShallowIterator, consumerTopicStats) (queue, stream) }) ).flatten.toList @@ -402,7 +405,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val myTopicThreadIdsMap = TopicCount.constructTopicCount(group, consumerIdString, zkClient).getConsumerThreadIdsPerTopic val consumersPerTopicMap = getConsumersPerTopic(zkClient, group) val brokers = getAllBrokersInCluster(zkClient) - val topicsMetadata = ClientUtils.fetchTopicMetadata(myTopicThreadIdsMap.keySet, brokers).topicsMetadata + val topicsMetadata = ClientUtils.fetchTopicMetadata(myTopicThreadIdsMap.keySet, brokers, config.clientId).topicsMetadata val partitionsPerTopicMap = new mutable.HashMap[String, Seq[Int]] val leaderIdForPartitionsMap = new mutable.HashMap[(String, Int), Int] topicsMetadata.foreach(m =>{ @@ -598,9 +601,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, case None => config.autoOffsetReset match { case OffsetRequest.SmallestTimeString => - SimpleConsumer.earliestOrLatestOffset(zkClient, topic, leader, partition, OffsetRequest.EarliestTime) + SimpleConsumer.earliestOrLatestOffset(zkClient, topic, leader, partition, OffsetRequest.EarliestTime, config.clientId) case OffsetRequest.LargestTimeString => - SimpleConsumer.earliestOrLatestOffset(zkClient, topic, leader, partition, OffsetRequest.LatestTime) + SimpleConsumer.earliestOrLatestOffset(zkClient, topic, leader, partition, OffsetRequest.LatestTime, config.clientId) case _ => throw new InvalidConfigException("Wrong value in autoOffsetReset in ConsumerConfig") } @@ -614,7 +617,8 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, queue, consumedOffset, fetchedOffset, - new AtomicInteger(config.fetchSize)) + new AtomicInteger(config.fetchSize), + consumerTopicStats) partTopicInfoMap.put(partition, partTopicInfo) debug(partTopicInfo + " selected new offset " + offset) } @@ -670,7 +674,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val q = e._2._1 topicThreadIdAndQueues.put(topicThreadId, q) newGauge( - config.groupId + "-" + topicThreadId._1 + "-" + topicThreadId._2 + "-FetchQueueSize", + config.clientId + "-" + config.groupId + "-" + topicThreadId._1 + "-" + topicThreadId._2 + "-FetchQueueSize", new Gauge[Int] { def getValue = q.size } @@ -717,7 +721,8 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, config.consumerTimeoutMs, keyDecoder, valueDecoder, - config.enableShallowIterator) + config.enableShallowIterator, + consumerTopicStats) (queue, stream) }).toList diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 3b1a53f..4840c0c 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -961,7 +961,7 @@ case class PartitionAndReplica(topic: String, partition: Int, replica: Int) case class LeaderIsrAndControllerEpoch(val leaderAndIsr: LeaderAndIsr, controllerEpoch: Int) -object ControllerStat extends KafkaMetricsGroup { +object ControllerStats extends KafkaMetricsGroup { val offlinePartitionRate = newMeter("OfflinePartitionsPerSec", "partitions", TimeUnit.SECONDS) val uncleanLeaderElectionRate = newMeter("UncleanLeaderElectionsPerSec", "elections", TimeUnit.SECONDS) val leaderElectionTimer = new KafkaTimer(newTimer("LeaderElectionRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS)) diff --git a/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala b/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala index 3e5435e..3eb23cd 100644 --- a/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala +++ b/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala @@ -58,12 +58,12 @@ class OfflinePartitionLeaderSelector(controllerContext: ControllerContext) exten .format(liveAssignedReplicasToThisPartition.mkString(","))) liveAssignedReplicasToThisPartition.isEmpty match { case true => - ControllerStat.offlinePartitionRate.mark() + ControllerStats.offlinePartitionRate.mark() throw new PartitionOfflineException(("No replica for partition " + "([%s, %d]) is alive. Live brokers are: [%s],".format(topic, partition, controllerContext.liveBrokerIds)) + " Assigned replicas are: [%s]".format(assignedReplicas)) case false => - ControllerStat.uncleanLeaderElectionRate.mark() + ControllerStats.uncleanLeaderElectionRate.mark() val newLeader = liveAssignedReplicasToThisPartition.head warn("No broker in ISR is alive, elected leader from the alive replicas is [%s], ".format(newLeader) + "There's potential data loss") @@ -78,7 +78,7 @@ class OfflinePartitionLeaderSelector(controllerContext: ControllerContext) exten partition)) (newLeaderAndIsr, liveAssignedReplicasToThisPartition) case None => - ControllerStat.offlinePartitionRate.mark() + ControllerStats.offlinePartitionRate.mark() throw new PartitionOfflineException("Partition [%s, %d] doesn't have".format(topic, partition) + "replicas assigned to it") } diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala index 17dfbee..0278782 100644 --- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala +++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala @@ -223,7 +223,7 @@ class PartitionStateMachine(controller: KafkaController) extends Logging { val liveAssignedReplicas = replicaAssignment.filter(r => controllerContext.liveBrokerIds.contains(r)) liveAssignedReplicas.size match { case 0 => - ControllerStat.offlinePartitionRate.mark() + ControllerStats.offlinePartitionRate.mark() throw new StateChangeFailedException(("During state change of partition %s from NEW to ONLINE, assigned replicas are " + "[%s], live brokers are [%s]. No assigned replica is alive").format(topicAndPartition, replicaAssignment.mkString(","), controllerContext.liveBrokerIds)) @@ -249,7 +249,7 @@ class PartitionStateMachine(controller: KafkaController) extends Logging { // read the controller epoch val leaderIsrAndEpoch = ZkUtils.getLeaderIsrAndEpochForPartition(zkClient, topicAndPartition.topic, topicAndPartition.partition).get - ControllerStat.offlinePartitionRate.mark() + ControllerStats.offlinePartitionRate.mark() throw new StateChangeFailedException("Error while changing partition %s's state from New to Online" .format(topicAndPartition) + " since Leader and isr path already exists with value " + "%s and controller epoch %d".format(leaderIsrAndEpoch.leaderAndIsr.toString(), leaderIsrAndEpoch.controllerEpoch)) diff --git a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala index 5c8aec5..1753947 100644 --- a/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala +++ b/core/src/main/scala/kafka/controller/ReplicaStateMachine.scala @@ -227,7 +227,7 @@ class ReplicaStateMachine(controller: KafkaController) extends Logging { class BrokerChangeListener() extends IZkChildListener with Logging { this.logIdent = "[BrokerChangeListener on Controller " + controller.config.brokerId + "]: " def handleChildChange(parentPath : String, currentBrokerList : java.util.List[String]) { - ControllerStat.leaderElectionTimer.time { + ControllerStats.leaderElectionTimer.time { info("Broker change listener fired for path %s with children %s".format(parentPath, currentBrokerList.mkString(","))) if(!isShuttingDown.get()) { controllerContext.controllerLock synchronized { diff --git a/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala b/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala index ad98b75..dbf04fd 100644 --- a/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala @@ -20,14 +20,15 @@ import kafka.api._ import java.nio.ByteBuffer import scala.collection.JavaConversions -class TopicMetadataRequest(val versionId: Short, +class TopicMetadataRequest(val correlationId: Int, + val versionId: Short, val clientId: String, val topics: java.util.List[String]) extends RequestOrResponse(Some(kafka.api.RequestKeys.MetadataKey)) { val underlying: kafka.api.TopicMetadataRequest = - new kafka.api.TopicMetadataRequest(versionId, clientId, JavaConversions.asBuffer(topics)) + new kafka.api.TopicMetadataRequest(versionId, clientId, JavaConversions.asBuffer(topics), correlationId) def this(topics: java.util.List[String]) = - this(kafka.api.TopicMetadataRequest.CurrentVersion, kafka.api.TopicMetadataRequest.DefaultClientId, topics) + this(0, kafka.api.TopicMetadataRequest.CurrentVersion, kafka.api.TopicMetadataRequest.DefaultClientId, topics) def writeTo(buffer: ByteBuffer) = underlying.writeTo(buffer) diff --git a/core/src/main/scala/kafka/javaapi/consumer/SimpleConsumer.scala b/core/src/main/scala/kafka/javaapi/consumer/SimpleConsumer.scala index 803ec4b..58c7081 100644 --- a/core/src/main/scala/kafka/javaapi/consumer/SimpleConsumer.scala +++ b/core/src/main/scala/kafka/javaapi/consumer/SimpleConsumer.scala @@ -29,8 +29,10 @@ import kafka.javaapi.OffsetRequest class SimpleConsumer(val host: String, val port: Int, val soTimeout: Int, - val bufferSize: Int) { - private val underlying = new kafka.consumer.SimpleConsumer(host, port, soTimeout, bufferSize) + val bufferSize: Int, + val clientId: String) { + + private val underlying = new kafka.consumer.SimpleConsumer(host, port, soTimeout, bufferSize, clientId) /** * Fetch a set of messages from a topic. This version of the fetch method diff --git a/core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala index 2b20aa4..14c4c8a 100644 --- a/core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala @@ -16,7 +16,6 @@ */ package kafka.javaapi.consumer -import kafka.message.Message import kafka.serializer._ import kafka.consumer._ import scala.collection.JavaConversions.asList diff --git a/core/src/main/scala/kafka/message/Message.scala b/core/src/main/scala/kafka/message/Message.scala index aedab42..12a8368 100644 --- a/core/src/main/scala/kafka/message/Message.scala +++ b/core/src/main/scala/kafka/message/Message.scala @@ -38,17 +38,20 @@ object Message { val KeySizeOffset = AttributesOffset + AttributesLength val KeySizeLength = 4 val KeyOffset = KeySizeOffset + KeySizeLength - val MessageOverhead = KeyOffset + val ValueSizeLength = 4 + + /** The amount of overhead bytes in a message */ + val MessageOverhead = KeyOffset + ValueSizeLength /** * The minimum valid size for the message header */ - val MinHeaderSize = CrcLength + MagicLength + AttributesLength + KeySizeLength + val MinHeaderSize = CrcLength + MagicLength + AttributesLength + KeySizeLength + ValueSizeLength /** * The current "magic" value */ - val CurrentMagicValue: Byte = 2 + val CurrentMagicValue: Byte = 0 /** * Specifies the mask for the compression code. 2 bits to hold the compression codec. @@ -97,22 +100,24 @@ class Message(val buffer: ByteBuffer) { Message.AttributesLength + Message.KeySizeLength + (if(key == null) 0 else key.length) + + Message.ValueSizeLength + (if(payloadSize >= 0) payloadSize else bytes.length - payloadOffset))) // skip crc, we will fill that in at the end - buffer.put(MagicOffset, CurrentMagicValue) - var attributes:Byte = 0 + buffer.position(MagicOffset) + buffer.put(CurrentMagicValue) + var attributes: Byte = 0 if (codec.codec > 0) attributes = (attributes | (CompressionCodeMask & codec.codec)).toByte - buffer.put(AttributesOffset, attributes) + buffer.put(attributes) if(key == null) { - buffer.putInt(KeySizeOffset, -1) - buffer.position(KeyOffset) + buffer.putInt(-1) } else { - buffer.putInt(KeySizeOffset, key.length) - buffer.position(KeyOffset) + buffer.putInt(key.length) buffer.put(key, 0, key.length) } - buffer.put(bytes, payloadOffset, if(payloadSize >= 0) payloadSize else bytes.length - payloadOffset) + val size = if(payloadSize >= 0) payloadSize else bytes.length - payloadOffset + buffer.putInt(size) + buffer.put(bytes, payloadOffset, size) buffer.rewind() // now compute the checksum and fill it in @@ -171,9 +176,14 @@ class Message(val buffer: ByteBuffer) { def hasKey: Boolean = keySize >= 0 /** + * The position where the payload size is stored + */ + private def payloadSizeOffset = Message.KeyOffset + max(0, keySize) + + /** * The length of the message value in bytes */ - def payloadSize: Int = size - KeyOffset - max(0, keySize) + def payloadSize: Int = buffer.getInt(payloadSizeOffset) /** * The magic version of this message @@ -194,29 +204,27 @@ class Message(val buffer: ByteBuffer) { /** * A ByteBuffer containing the content of the message */ - def payload: ByteBuffer = { - var payload = buffer.duplicate - payload.position(KeyOffset + max(keySize, 0)) - payload = payload.slice() - payload.limit(payloadSize) - payload.rewind() - payload - } + def payload: ByteBuffer = sliceDelimited(payloadSizeOffset) /** * A ByteBuffer containing the message key */ - def key: ByteBuffer = { - val s = keySize - if(s < 0) { + def key: ByteBuffer = sliceDelimited(KeySizeOffset) + + /** + * Read a size-delimited byte buffer starting at the given offset + */ + private def sliceDelimited(start: Int): ByteBuffer = { + val size = buffer.getInt(start) + if(size < 0) { null } else { - var key = buffer.duplicate - key.position(KeyOffset) - key = key.slice() - key.limit(s) - key.rewind() - key + var b = buffer.duplicate + b.position(start + 4) + b = b.slice() + b.limit(size) + b.rewind + b } } diff --git a/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala b/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala index fda0b24..ea9559f 100644 --- a/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala +++ b/core/src/main/scala/kafka/metrics/KafkaCSVMetricsReporter.scala @@ -24,7 +24,6 @@ import com.yammer.metrics.Metrics import java.io.File import com.yammer.metrics.reporting.CsvReporter import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean import kafka.utils.{Utils, VerifiableProperties, Logging} diff --git a/core/src/main/scala/kafka/producer/BrokerPartitionInfo.scala b/core/src/main/scala/kafka/producer/BrokerPartitionInfo.scala index 27bae5e..d58a063 100644 --- a/core/src/main/scala/kafka/producer/BrokerPartitionInfo.scala +++ b/core/src/main/scala/kafka/producer/BrokerPartitionInfo.scala @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ package kafka.producer import collection.mutable.HashMap @@ -68,11 +68,11 @@ class BrokerPartitionInfo(producerConfig: ProducerConfig, /** * It updates the cache by issuing a get topic metadata request to a random broker. - * @param topic the topic for which the metadata is to be fetched + * @param topics the topics for which the metadata is to be fetched */ - def updateInfo(topics: Set[String]) = { + def updateInfo(topics: Set[String]) { var topicsMetadata: Seq[TopicMetadata] = Nil - val topicMetadataResponse = ClientUtils.fetchTopicMetadata(topics, brokers) + val topicMetadataResponse = ClientUtils.fetchTopicMetadata(topics, brokers, producerConfig) topicsMetadata = topicMetadataResponse.topicsMetadata // throw partition specific exception topicsMetadata.foreach(tmd =>{ diff --git a/core/src/main/scala/kafka/producer/ConsoleProducer.scala b/core/src/main/scala/kafka/producer/ConsoleProducer.scala index 8664cb1..8c32115 100644 --- a/core/src/main/scala/kafka/producer/ConsoleProducer.scala +++ b/core/src/main/scala/kafka/producer/ConsoleProducer.scala @@ -21,6 +21,7 @@ import scala.collection.JavaConversions._ import joptsimple._ import java.util.Properties import java.io._ +import kafka.common._ import kafka.message._ import kafka.serializer._ @@ -49,13 +50,18 @@ object ConsoleProducer { .describedAs("timeout_ms") .ofType(classOf[java.lang.Long]) .defaultsTo(1000) - val messageEncoderOpt = parser.accepts("message-encoder", "The class name of the message encoder implementation to use.") + val valueEncoderOpt = parser.accepts("value-serializer", "The class name of the message encoder implementation to use for serializing values.") + .withRequiredArg + .describedAs("encoder_class") + .ofType(classOf[java.lang.String]) + .defaultsTo(classOf[StringEncoder].getName) + val keyEncoderOpt = parser.accepts("key-serializer", "The class name of the message encoder implementation to use for serializing keys.") .withRequiredArg .describedAs("encoder_class") .ofType(classOf[java.lang.String]) .defaultsTo(classOf[StringEncoder].getName) val messageReaderOpt = parser.accepts("line-reader", "The class name of the class to use for reading lines from standard in. " + - "By default each line is read as a seperate message.") + "By default each line is read as a separate message.") .withRequiredArg .describedAs("reader_class") .ofType(classOf[java.lang.String]) @@ -82,9 +88,11 @@ object ConsoleProducer { val compress = options.has(compressOpt) val batchSize = options.valueOf(batchSizeOpt) val sendTimeout = options.valueOf(sendTimeoutOpt) - val encoderClass = options.valueOf(messageEncoderOpt) + val keyEncoderClass = options.valueOf(keyEncoderOpt) + val valueEncoderClass = options.valueOf(valueEncoderOpt) val readerClass = options.valueOf(messageReaderOpt) val cmdLineProps = parseLineReaderArgs(options.valuesOf(propertyOpt)) + cmdLineProps.put("topic", topic) val props = new Properties() props.put("broker.list", brokerList) @@ -94,12 +102,13 @@ object ConsoleProducer { if(options.has(batchSizeOpt)) props.put("batch.size", batchSize.toString) props.put("queue.time", sendTimeout.toString) - props.put("serializer.class", encoderClass) + props.put("key.serializer.class", keyEncoderClass) + props.put("serializer.class", valueEncoderClass) - val reader = Class.forName(readerClass).newInstance().asInstanceOf[MessageReader] + val reader = Class.forName(readerClass).newInstance().asInstanceOf[MessageReader[AnyRef, AnyRef]] reader.init(System.in, cmdLineProps) - val producer = new Producer[Any, Any](new ProducerConfig(props)) + val producer = new Producer[AnyRef, AnyRef](new ProducerConfig(props)) Runtime.getRuntime.addShutdownHook(new Thread() { override def run() { @@ -107,11 +116,11 @@ object ConsoleProducer { } }) - var message: AnyRef = null + var message: KeyedMessage[AnyRef, AnyRef] = null do { message = reader.readMessage() if(message != null) - producer.send(new KeyedMessage(topic, message)) + producer.send(message) } while(message != null) } @@ -127,19 +136,49 @@ object ConsoleProducer { props } - trait MessageReader { + trait MessageReader[K,V] { def init(inputStream: InputStream, props: Properties) {} - def readMessage(): AnyRef + def readMessage(): KeyedMessage[K,V] def close() {} } - class LineMessageReader extends MessageReader { + class LineMessageReader extends MessageReader[String, String] { + var topic: String = null var reader: BufferedReader = null - - override def init(inputStream: InputStream, props: Properties) { + var parseKey = false + var keySeparator = "\t" + var ignoreError = false + var lineNumber = 0 + + override def init(inputStream: InputStream, props: Properties) { + topic = props.getProperty("topic") + if(props.containsKey("parse.key")) + parseKey = props.getProperty("parse.key").trim.toLowerCase.equals("true") + if(props.containsKey("key.seperator")) + keySeparator = props.getProperty("key.separator") + if(props.containsKey("ignore.error")) + ignoreError = props.getProperty("ignore.error").trim.toLowerCase.equals("true") reader = new BufferedReader(new InputStreamReader(inputStream)) } - override def readMessage() = reader.readLine() + override def readMessage() = { + lineNumber += 1 + val line = reader.readLine() + if(parseKey) { + line.indexOf(keySeparator) match { + case -1 => + if(ignoreError) + new KeyedMessage(topic, line) + else + throw new KafkaException("No key found on line " + lineNumber + ": " + line) + case n => + new KeyedMessage(topic, + line.substring(0, n), + if(n + keySeparator.size > line.size) "" else line.substring(n + keySeparator.size)) + } + } else { + new KeyedMessage(topic, line) + } + } } } diff --git a/core/src/main/scala/kafka/producer/DefaultPartitioner.scala b/core/src/main/scala/kafka/producer/DefaultPartitioner.scala index d0a89eb..c82670e 100644 --- a/core/src/main/scala/kafka/producer/DefaultPartitioner.scala +++ b/core/src/main/scala/kafka/producer/DefaultPartitioner.scala @@ -17,7 +17,6 @@ package kafka.producer -import kafka.utils.Utils import kafka.utils._ diff --git a/core/src/main/scala/kafka/producer/Producer.scala b/core/src/main/scala/kafka/producer/Producer.scala index 7b8926c..3bfd563 100644 --- a/core/src/main/scala/kafka/producer/Producer.scala +++ b/core/src/main/scala/kafka/producer/Producer.scala @@ -16,7 +16,7 @@ */ package kafka.producer -import async.{AsyncProducerStats, DefaultEventHandler, ProducerSendThread, EventHandler} +import async.{DefaultEventHandler, ProducerSendThread, EventHandler} import kafka.utils._ import java.util.Random import java.util.concurrent.{TimeUnit, LinkedBlockingQueue} @@ -27,8 +27,11 @@ import kafka.metrics._ class Producer[K,V](config: ProducerConfig, - private val eventHandler: EventHandler[K,V]) // for testing only -extends Logging { + private val eventHandler: EventHandler[K,V], + private val producerStats: ProducerStats, + private val producerTopicStats: ProducerTopicStats) // only for unit testing + extends Logging { + private val hasShutdown = new AtomicBoolean(false) if (config.batchSize > config.queueSize) throw new InvalidConfigException("Batch size can't be larger than queue size.") @@ -47,25 +50,38 @@ extends Logging { queue, eventHandler, config.queueTime, - config.batchSize) + config.batchSize, + config.clientId) producerSendThread.start() case _ => throw new InvalidConfigException("Valid values for producer.type are sync/async") } KafkaMetricsReporter.startReporters(config.props) + def this(t: (ProducerConfig, EventHandler[K,V], ProducerStats, ProducerTopicStats)) = + this(t._1, t._2, t._3, t._4) + def this(config: ProducerConfig) = - this(config, - new DefaultEventHandler[K,V](config, - Utils.createObject[Partitioner[K]](config.partitionerClass, config.props), - Utils.createObject[Encoder[V]](config.serializerClass, config.props), - Utils.createObject[Encoder[K]](config.keySerializerClass, config.props), - new ProducerPool(config))) + this { + ClientId.validate(config.clientId) + val producerStats = new ProducerStats(config.clientId) + val producerTopicStats = new ProducerTopicStats(config.clientId) + (config, + new DefaultEventHandler[K,V](config, + Utils.createObject[Partitioner[K]](config.partitionerClass, config.props), + Utils.createObject[Encoder[V]](config.serializerClass, config.props), + Utils.createObject[Encoder[K]](config.keySerializerClass, config.props), + new ProducerPool(config), + producerStats = producerStats, + producerTopicStats = producerTopicStats), + producerStats, + producerTopicStats) + } /** * Sends the data, partitioned by key to the topic using either the * synchronous or the asynchronous producer - * @param producerData the producer data object that encapsulates the topic, key and message data + * @param messages the producer data object that encapsulates the topic, key and message data */ def send(messages: KeyedMessage[K,V]*) { if (hasShutdown.get) @@ -79,8 +95,8 @@ extends Logging { private def recordStats(messages: Seq[KeyedMessage[K,V]]) { for (message <- messages) { - ProducerTopicStat.getProducerTopicStat(message.topic).messageRate.mark() - ProducerTopicStat.getProducerAllTopicStat.messageRate.mark() + producerTopicStats.getProducerTopicStats(message.topic).messageRate.mark() + producerTopicStats.getProducerAllTopicStats.messageRate.mark() } } @@ -105,7 +121,7 @@ extends Logging { } } if(!added) { - AsyncProducerStats.droppedMessageRate.mark() + producerStats.droppedMessageRate.mark() error("Event queue is full of unsent messages, could not send event: " + message.toString) throw new QueueFullException("Event queue is full of unsent messages, could not send event: " + message.toString) }else { @@ -131,26 +147,27 @@ extends Logging { } @threadsafe -class ProducerTopicStat(name: String) extends KafkaMetricsGroup { - val messageRate = newMeter(name + "MessagesPerSec", "messages", TimeUnit.SECONDS) - val byteRate = newMeter(name + "BytesPerSec", "bytes", TimeUnit.SECONDS) +class ProducerTopicMetrics(clientIdTopic: ClientIdAndTopic) extends KafkaMetricsGroup { + val messageRate = newMeter(clientIdTopic + "-MessagesPerSec", "messages", TimeUnit.SECONDS) + val byteRate = newMeter(clientIdTopic + "-BytesPerSec", "bytes", TimeUnit.SECONDS) } -object ProducerTopicStat { - private val valueFactory = (k: String) => new ProducerTopicStat(k) - private val stats = new Pool[String, ProducerTopicStat](Some(valueFactory)) - private val allTopicStat = new ProducerTopicStat("AllTopics") +class ProducerTopicStats(clientId: String) { + private val valueFactory = (k: ClientIdAndTopic) => new ProducerTopicMetrics(k) + private val stats = new Pool[ClientIdAndTopic, ProducerTopicMetrics](Some(valueFactory)) + private val allTopicStats = new ProducerTopicMetrics(new ClientIdAndTopic(clientId, "AllTopics")) - def getProducerAllTopicStat(): ProducerTopicStat = allTopicStat + def getProducerAllTopicStats(): ProducerTopicMetrics = allTopicStats - def getProducerTopicStat(topic: String): ProducerTopicStat = { - stats.getAndMaybePut(topic + "-") + def getProducerTopicStats(topic: String): ProducerTopicMetrics = { + stats.getAndMaybePut(new ClientIdAndTopic(clientId, topic)) } } -object ProducerStats extends KafkaMetricsGroup { - val serializationErrorRate = newMeter("SerializationErrorsPerSec", "errors", TimeUnit.SECONDS) - val resendRate = newMeter( "ResendsPerSec", "resends", TimeUnit.SECONDS) - val failedSendRate = newMeter("FailedSendsPerSec", "failed sends", TimeUnit.SECONDS) +class ProducerStats(clientId: String) extends KafkaMetricsGroup { + val serializationErrorRate = newMeter(clientId + "-SerializationErrorsPerSec", "errors", TimeUnit.SECONDS) + val resendRate = newMeter(clientId + "-ResendsPerSec", "resends", TimeUnit.SECONDS) + val failedSendRate = newMeter(clientId + "-FailedSendsPerSec", "failed sends", TimeUnit.SECONDS) + val droppedMessageRate = newMeter(clientId + "-DroppedMessagesPerSec", "drops", TimeUnit.SECONDS) } diff --git a/core/src/main/scala/kafka/producer/ProducerConfig.scala b/core/src/main/scala/kafka/producer/ProducerConfig.scala index 72f68dc..cde0f9a 100644 --- a/core/src/main/scala/kafka/producer/ProducerConfig.scala +++ b/core/src/main/scala/kafka/producer/ProducerConfig.scala @@ -33,7 +33,7 @@ class ProducerConfig private (val props: VerifiableProperties) /** This is for bootstrapping and the producer will only use it for getting metadata * (topics, partitions and replicas). The socket connections for sending the actual data * will be established based on the broker information returned in the metadata. The - * format is host1:por1,host2:port2, and the list can be a subset of brokers or + * format is host1:port1,host2:port2, and the list can be a subset of brokers or * a VIP pointing to a subset of brokers. */ val brokerList = props.getString("broker.list") diff --git a/core/src/main/scala/kafka/producer/ProducerPool.scala b/core/src/main/scala/kafka/producer/ProducerPool.scala index eb8ead3..4970029 100644 --- a/core/src/main/scala/kafka/producer/ProducerPool.scala +++ b/core/src/main/scala/kafka/producer/ProducerPool.scala @@ -26,13 +26,15 @@ import kafka.api.TopicMetadata import kafka.common.UnavailableProducerException -object ProducerPool{ - def createSyncProducer(configOpt: Option[ProducerConfig], broker: Broker): SyncProducer = { +object ProducerPool { + /** + * Used in ProducerPool to initiate a SyncProducer connection with a broker. + */ + def createSyncProducer(config: ProducerConfig, broker: Broker): SyncProducer = { val props = new Properties() props.put("host", broker.host) props.put("port", broker.port.toString) - if(configOpt.isDefined) - props.putAll(configOpt.get.props.props) + props.putAll(config.props.props) new SyncProducer(new SyncProducerConfig(props)) } } @@ -41,9 +43,9 @@ class ProducerPool(val config: ProducerConfig) extends Logging { private val syncProducers = new HashMap[Int, SyncProducer] private val lock = new Object() - def updateProducer(topicMetaDatas: Seq[TopicMetadata]) { + def updateProducer(topicMetadatas: Seq[TopicMetadata]) { val newBrokers = new collection.mutable.HashSet[Broker] - topicMetaDatas.foreach(tmd => { + topicMetadatas.foreach(tmd => { tmd.partitionsMetadata.foreach(pmd => { if(pmd.leader.isDefined) newBrokers+=(pmd.leader.get) @@ -53,9 +55,9 @@ class ProducerPool(val config: ProducerConfig) extends Logging { newBrokers.foreach(b => { if(syncProducers.contains(b.id)){ syncProducers(b.id).close() - syncProducers.put(b.id, ProducerPool.createSyncProducer(Some(config), b)) + syncProducers.put(b.id, ProducerPool.createSyncProducer(config, b)) } else - syncProducers.put(b.id, ProducerPool.createSyncProducer(Some(config), b)) + syncProducers.put(b.id, ProducerPool.createSyncProducer(config, b)) }) } } diff --git a/core/src/main/scala/kafka/producer/SyncProducer.scala b/core/src/main/scala/kafka/producer/SyncProducer.scala index 15733bb..3183ceb 100644 --- a/core/src/main/scala/kafka/producer/SyncProducer.scala +++ b/core/src/main/scala/kafka/producer/SyncProducer.scala @@ -34,14 +34,12 @@ object SyncProducer { */ @threadsafe class SyncProducer(val config: SyncProducerConfig) extends Logging { - - private val MaxConnectBackoffMs = 60000 - private var sentOnConnection = 0 private val lock = new Object() @volatile private var shutdown: Boolean = false private val blockingChannel = new BlockingChannel(config.host, config.port, BlockingChannel.UseDefaultBufferSize, config.bufferSize, config.requestTimeoutMs) + val producerRequestStats = new ProducerRequestStats(config.clientId + "-host_%s-port_%s".format(config.host, config.port)) trace("Instantiating Scala Sync Producer") @@ -89,9 +87,9 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { * Send a message */ def send(producerRequest: ProducerRequest): ProducerResponse = { - ProducerRequestStat.requestSizeHist.update(producerRequest.sizeInBytes) + producerRequestStats.requestSizeHist.update(producerRequest.sizeInBytes) var response: Receive = null - ProducerRequestStat.requestTimer.time { + producerRequestStats.requestTimer.time { response = doSend(producerRequest) } ProducerResponse.readFrom(response.buffer) @@ -152,7 +150,7 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { } } -object ProducerRequestStat extends KafkaMetricsGroup { - val requestTimer = new KafkaTimer(newTimer("ProduceRequestRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS)) - val requestSizeHist = newHistogram("ProducerRequestSize") +class ProducerRequestStats(clientId: String) extends KafkaMetricsGroup { + val requestTimer = new KafkaTimer(newTimer(clientId + "-ProduceRequestRateAndTimeMs", TimeUnit.MILLISECONDS, TimeUnit.SECONDS)) + val requestSizeHist = newHistogram(clientId + "-ProducerRequestSize") } diff --git a/core/src/main/scala/kafka/producer/SyncProducerConfig.scala b/core/src/main/scala/kafka/producer/SyncProducerConfig.scala index 6f6a3f3..5ebd29a 100644 --- a/core/src/main/scala/kafka/producer/SyncProducerConfig.scala +++ b/core/src/main/scala/kafka/producer/SyncProducerConfig.scala @@ -41,10 +41,7 @@ trait SyncProducerConfigShared { val maxMessageSize = props.getInt("max.message.size", 1000000) /* the client application sending the producer requests */ - val correlationId = props.getInt("producer.request.correlation_id", SyncProducerConfig.DefaultCorrelationId) - - /* the client application sending the producer requests */ - val clientId = props.getString("producer.request.client_id",SyncProducerConfig.DefaultClientId) + val clientId = props.getString("clientid", SyncProducerConfig.DefaultClientId) /* * The required acks of the producer requests - negative value means ack @@ -61,8 +58,7 @@ trait SyncProducerConfigShared { } object SyncProducerConfig { - val DefaultCorrelationId = -1 val DefaultClientId = "" val DefaultRequiredAcks : Short = 0 - val DefaultAckTimeoutMs = 500 + val DefaultAckTimeoutMs = 1500 } \ No newline at end of file diff --git a/core/src/main/scala/kafka/producer/async/AsyncProducerStats.scala b/core/src/main/scala/kafka/producer/async/AsyncProducerStats.scala index dd9078e..e69de29 100644 --- a/core/src/main/scala/kafka/producer/async/AsyncProducerStats.scala +++ b/core/src/main/scala/kafka/producer/async/AsyncProducerStats.scala @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package kafka.producer.async - -import kafka.metrics.KafkaMetricsGroup -import java.util.concurrent.TimeUnit - -object AsyncProducerStats extends KafkaMetricsGroup { - val droppedMessageRate = newMeter("DroppedMessagesPerSec", "drops", TimeUnit.SECONDS) -} diff --git a/core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala b/core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala index a84be2d..7d0f609 100644 --- a/core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala +++ b/core/src/main/scala/kafka/producer/async/DefaultEventHandler.scala @@ -33,11 +33,14 @@ class DefaultEventHandler[K,V](config: ProducerConfig, private val encoder: Encoder[V], private val keyEncoder: Encoder[K], private val producerPool: ProducerPool, - private val topicPartitionInfos: HashMap[String, TopicMetadata] = new HashMap[String, TopicMetadata]) + private val topicPartitionInfos: HashMap[String, TopicMetadata] = new HashMap[String, TopicMetadata], + private val producerStats: ProducerStats, + private val producerTopicStats: ProducerTopicStats) extends EventHandler[K,V] with Logging { val isSync = ("sync" == config.producerType) - val counter = new AtomicInteger(0) + val partitionCounter = new AtomicInteger(0) + val correlationCounter = new AtomicInteger(0) val brokerPartitionInfo = new BrokerPartitionInfo(config, producerPool, topicPartitionInfos) private val lock = new Object() @@ -46,10 +49,10 @@ class DefaultEventHandler[K,V](config: ProducerConfig, lock synchronized { val serializedData = serialize(events) serializedData.foreach{ - keyed => + keyed => val dataSize = keyed.message.payloadSize - ProducerTopicStat.getProducerTopicStat(keyed.topic).byteRate.mark(dataSize) - ProducerTopicStat.getProducerAllTopicStat.byteRate.mark(dataSize) + producerTopicStats.getProducerTopicStats(keyed.topic).byteRate.mark(dataSize) + producerTopicStats.getProducerAllTopicStats.byteRate.mark(dataSize) } var outstandingProduceRequests = serializedData var remainingRetries = config.producerRetries + 1 @@ -61,11 +64,11 @@ class DefaultEventHandler[K,V](config: ProducerConfig, // get topics of the outstanding produce requests and refresh metadata for those Utils.swallowError(brokerPartitionInfo.updateInfo(outstandingProduceRequests.map(_.topic).toSet)) remainingRetries -= 1 - ProducerStats.resendRate.mark() + producerStats.resendRate.mark() } } if(outstandingProduceRequests.size > 0) { - ProducerStats.failedSendRate.mark() + producerStats.failedSendRate.mark() error("Failed to send the following requests: " + outstandingProduceRequests) throw new FailedToSendMessageException("Failed to send messages after " + config.producerRetries + " tries.", null) } @@ -80,7 +83,7 @@ class DefaultEventHandler[K,V](config: ProducerConfig, try { for ((brokerid, messagesPerBrokerMap) <- partitionedData) { if (logger.isTraceEnabled) - messagesPerBrokerMap.foreach(partitionAndEvent => + messagesPerBrokerMap.foreach(partitionAndEvent => trace("Handling event for Topic: %s, Broker: %d, Partitions: %s".format(partitionAndEvent._1, brokerid, partitionAndEvent._2))) val messageSetPerBroker = groupMessagesToSet(messagesPerBrokerMap) @@ -111,7 +114,7 @@ class DefaultEventHandler[K,V](config: ProducerConfig, serializedMessages += KeyedMessage[K,Message](topic = e.topic, key = null.asInstanceOf[K], message = new Message(bytes = encoder.toBytes(e.message))) } catch { case t => - ProducerStats.serializationErrorRate.mark() + producerStats.serializationErrorRate.mark() if (isSync) { throw t } else { @@ -171,7 +174,7 @@ class DefaultEventHandler[K,V](config: ProducerConfig, debug("Broker partitions registered for topic: %s are %s" .format(m.topic, topicPartitionsList.map(p => p.partitionId).mkString(","))) val totalNumPartitions = topicPartitionsList.length - if(totalNumPartitions == 0) + if(totalNumPartitions == 0) throw new NoBrokersForPartitionException("Partition key = " + m.key) topicPartitionsList } @@ -187,10 +190,10 @@ class DefaultEventHandler[K,V](config: ProducerConfig, if(numPartitions <= 0) throw new UnknownTopicOrPartitionException("Invalid number of partitions: " + numPartitions + "\n Valid values are > 0") - val partition = - if(key == null) - Utils.abs(counter.getAndIncrement()) % numPartitions - else + val partition = + if(key == null) + Utils.abs(partitionCounter.getAndIncrement()) % numPartitions + else partitioner.partition(key, numPartitions) if(partition < 0 || partition >= numPartitions) throw new UnknownTopicOrPartitionException("Invalid partition id : " + partition + @@ -210,20 +213,21 @@ class DefaultEventHandler[K,V](config: ProducerConfig, warn("Failed to send to broker %d with data %s".format(brokerId, messagesPerTopic)) messagesPerTopic.keys.toSeq } else if(messagesPerTopic.size > 0) { - val topicPartitionDataPairs = messagesPerTopic.toSeq.map { - case (topicAndPartition, messages) => - (topicAndPartition, messages) - } - val producerRequest = new ProducerRequest(config.correlationId, config.clientId, config.requiredAcks, - config.requestTimeoutMs, Map(topicPartitionDataPairs:_*)) + val producerRequest = new ProducerRequest(correlationCounter.getAndIncrement(), config.clientId, config.requiredAcks, + config.requestTimeoutMs, messagesPerTopic) try { val syncProducer = producerPool.getProducer(brokerId) val response = syncProducer.send(producerRequest) - trace("Producer sent messages for topics %s to broker %d on %s:%d" + debug("Producer sent messages for topics %s to broker %d on %s:%d" .format(messagesPerTopic, brokerId, syncProducer.config.host, syncProducer.config.port)) if (response.status.size != producerRequest.data.size) throw new KafkaException("Incomplete response (%s) for producer request (%s)" - .format(response, producerRequest)) + .format(response, producerRequest)) + if (logger.isTraceEnabled) { + val successfullySentData = response.status.filter(_._2.error == ErrorMapping.NoError) + successfullySentData.foreach(m => messagesPerTopic(m._1).foreach(message => + trace("Successfully sent message: %s".format(Utils.readString(message.message.payload))))) + } response.status.filter(_._2.error != ErrorMapping.NoError).toSeq .map(partitionStatus => partitionStatus._1) } catch { @@ -238,33 +242,33 @@ class DefaultEventHandler[K,V](config: ProducerConfig, private def groupMessagesToSet(messagesPerTopicAndPartition: Map[TopicAndPartition, Seq[KeyedMessage[K,Message]]]) = { /** enforce the compressed.topics config here. - * If the compression codec is anything other than NoCompressionCodec, - * Enable compression only for specified topics if any - * If the list of compressed topics is empty, then enable the specified compression codec for all topics - * If the compression codec is NoCompressionCodec, compression is disabled for all topics - */ + * If the compression codec is anything other than NoCompressionCodec, + * Enable compression only for specified topics if any + * If the list of compressed topics is empty, then enable the specified compression codec for all topics + * If the compression codec is NoCompressionCodec, compression is disabled for all topics + */ val messagesPerTopicPartition = messagesPerTopicAndPartition.map { case (topicAndPartition, messages) => val rawMessages = messages.map(_.message) ( topicAndPartition, config.compressionCodec match { case NoCompressionCodec => - trace("Sending %d messages with no compression to %s".format(messages.size, topicAndPartition)) + debug("Sending %d messages with no compression to %s".format(messages.size, topicAndPartition)) new ByteBufferMessageSet(NoCompressionCodec, rawMessages: _*) case _ => config.compressedTopics.size match { case 0 => - trace("Sending %d messages with compression codec %d to %s" + debug("Sending %d messages with compression codec %d to %s" .format(messages.size, config.compressionCodec.codec, topicAndPartition)) new ByteBufferMessageSet(config.compressionCodec, rawMessages: _*) case _ => if(config.compressedTopics.contains(topicAndPartition.topic)) { - trace("Sending %d messages with compression codec %d to %s" + debug("Sending %d messages with compression codec %d to %s" .format(messages.size, config.compressionCodec.codec, topicAndPartition)) new ByteBufferMessageSet(config.compressionCodec, rawMessages: _*) } else { - trace("Sending %d messages to %s with no compression as it is not in compressed.topics - %s" + debug("Sending %d messages to %s with no compression as it is not in compressed.topics - %s" .format(messages.size, topicAndPartition, config.compressedTopics.toString)) new ByteBufferMessageSet(NoCompressionCodec, rawMessages: _*) } diff --git a/core/src/main/scala/kafka/producer/async/ProducerSendThread.scala b/core/src/main/scala/kafka/producer/async/ProducerSendThread.scala index 46ea7d4..2b39cab 100644 --- a/core/src/main/scala/kafka/producer/async/ProducerSendThread.scala +++ b/core/src/main/scala/kafka/producer/async/ProducerSendThread.scala @@ -19,7 +19,7 @@ package kafka.producer.async import kafka.utils.{SystemTime, Logging} import java.util.concurrent.{TimeUnit, CountDownLatch, BlockingQueue} -import collection.mutable.ListBuffer +import collection.mutable.ArrayBuffer import kafka.producer.KeyedMessage import kafka.metrics.KafkaMetricsGroup import com.yammer.metrics.core.Gauge @@ -28,12 +28,13 @@ class ProducerSendThread[K,V](val threadName: String, val queue: BlockingQueue[KeyedMessage[K,V]], val handler: EventHandler[K,V], val queueTime: Long, - val batchSize: Int) extends Thread(threadName) with Logging with KafkaMetricsGroup { + val batchSize: Int, + val clientId: String) extends Thread(threadName) with Logging with KafkaMetricsGroup { private val shutdownLatch = new CountDownLatch(1) private val shutdownCommand = new KeyedMessage[K,V]("shutdown", null.asInstanceOf[K], null.asInstanceOf[V]) - newGauge("ProducerQueueSize-" + getId, + newGauge(clientId + "-ProducerQueueSize-" + getId, new Gauge[Int] { def getValue = queue.size }) @@ -57,7 +58,7 @@ class ProducerSendThread[K,V](val threadName: String, private def processEvents() { var lastSend = SystemTime.milliseconds - var events = new ListBuffer[KeyedMessage[K,V]] + var events = new ArrayBuffer[KeyedMessage[K,V]] var full: Boolean = false // drain the queue until you get a shutdown command @@ -85,7 +86,7 @@ class ProducerSendThread[K,V](val threadName: String, // if either queue time has reached or batch size has reached, dispatch to event handler tryToHandle(events) lastSend = SystemTime.milliseconds - events = new ListBuffer[KeyedMessage[K,V]] + events = new ArrayBuffer[KeyedMessage[K,V]] } } // send the last batch of events diff --git a/core/src/main/scala/kafka/serializer/Decoder.scala b/core/src/main/scala/kafka/serializer/Decoder.scala index 9da8370..54d0b93 100644 --- a/core/src/main/scala/kafka/serializer/Decoder.scala +++ b/core/src/main/scala/kafka/serializer/Decoder.scala @@ -17,7 +17,6 @@ package kafka.serializer -import kafka.message._ import kafka.utils.VerifiableProperties /** @@ -37,13 +36,6 @@ class DefaultDecoder(props: VerifiableProperties = null) extends Decoder[Array[B } /** - * Decode messages without any key - */ -class KeylessMessageDecoder(props: VerifiableProperties = null) extends Decoder[Message] { - def fromBytes(bytes: Array[Byte]) = new Message(bytes) -} - -/** * The string encoder translates strings into bytes. It uses UTF8 by default but takes * an optional property serializer.encoding to control this. */ diff --git a/core/src/main/scala/kafka/serializer/Encoder.scala b/core/src/main/scala/kafka/serializer/Encoder.scala index fa9067f..020e73c 100644 --- a/core/src/main/scala/kafka/serializer/Encoder.scala +++ b/core/src/main/scala/kafka/serializer/Encoder.scala @@ -18,8 +18,6 @@ package kafka.serializer import kafka.utils.VerifiableProperties -import kafka.message._ -import kafka.utils.Utils /** * An encoder is a method of turning objects into byte arrays. diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index c2cc3cb..6d73c82 100644 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -27,7 +27,7 @@ import kafka.api.{FetchResponse, FetchResponsePartitionData, FetchRequestBuilder import kafka.metrics.KafkaMetricsGroup import com.yammer.metrics.core.Gauge import java.util.concurrent.atomic.AtomicLong -import kafka.utils.{Pool, ShutdownableThread} +import kafka.utils.{ClientIdAndTopic, Pool, ShutdownableThread} import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantLock @@ -38,12 +38,14 @@ import java.util.concurrent.locks.ReentrantLock abstract class AbstractFetcherThread(name: String, clientId: String, sourceBroker: Broker, socketTimeout: Int, socketBufferSize: Int, fetchSize: Int, fetcherBrokerId: Int = -1, maxWait: Int = 0, minBytes: Int = 1) extends ShutdownableThread(name) { - private val partitionMap = new mutable.HashMap[TopicAndPartition, Long] // a (topic, partition) -> offset map private val partitionMapLock = new ReentrantLock private val partitionMapCond = partitionMapLock.newCondition() - val simpleConsumer = new SimpleConsumer(sourceBroker.host, sourceBroker.port, socketTimeout, socketBufferSize) - val fetcherMetrics = FetcherStat.getFetcherStat(name + "-" + sourceBroker.id) + val simpleConsumer = new SimpleConsumer(sourceBroker.host, sourceBroker.port, socketTimeout, socketBufferSize, clientId) + private val brokerInfo = "host_%s-port_%s".format(sourceBroker.host, sourceBroker.port) + val fetcherStats = new FetcherStats(clientId + "-" + brokerInfo) + val fetcherMetrics = fetcherStats.getFetcherStats(name + "-" + sourceBroker.id) + val fetcherLagStats = new FetcherLagStats(clientId + "-" + brokerInfo) /* callbacks to be defined in subclass */ @@ -64,7 +66,7 @@ abstract class AbstractFetcherThread(name: String, clientId: String, sourceBroke override def doWork() { val fetchRequestuilder = new FetchRequestBuilder(). - clientId(clientId). + clientId(clientId + "-" + brokerInfo). replicaId(fetcherBrokerId). maxWait(maxWait). minBytes(minBytes) @@ -117,7 +119,7 @@ abstract class AbstractFetcherThread(name: String, clientId: String, sourceBroke case None => currentOffset.get } partitionMap.put(topicAndPartition, newOffset) - FetcherLagMetrics.getFetcherLagMetrics(topic, partitionId).lag = partitionData.hw - newOffset + fetcherLagStats.getFetcherLagStats(topic, partitionId).lag = partitionData.hw - newOffset fetcherMetrics.byteRate.mark(validBytes) // Once we hand off the partition data to the subclass, we can't mess with it any more in this thread processPartitionData(topicAndPartition, currentOffset.get, partitionData) @@ -182,10 +184,10 @@ abstract class AbstractFetcherThread(name: String, clientId: String, sourceBroke } } -class FetcherLagMetrics(name: (String, Int)) extends KafkaMetricsGroup { +class FetcherLagMetrics(clientIdTopicPartition: ClientIdTopicPartition) extends KafkaMetricsGroup { private[this] var lagVal = new AtomicLong(-1L) newGauge( - name._1 + "-" + name._2 + "-ConsumerLag", + clientIdTopicPartition + "-ConsumerLag", new Gauge[Long] { def getValue = lagVal.get } @@ -198,25 +200,29 @@ class FetcherLagMetrics(name: (String, Int)) extends KafkaMetricsGroup { def lag = lagVal.get } -object FetcherLagMetrics { - private val valueFactory = (k: (String, Int)) => new FetcherLagMetrics(k) - private val stats = new Pool[(String, Int), FetcherLagMetrics](Some(valueFactory)) +class FetcherLagStats(clientId: String) { + private val valueFactory = (k: ClientIdTopicPartition) => new FetcherLagMetrics(k) + private val stats = new Pool[ClientIdTopicPartition, FetcherLagMetrics](Some(valueFactory)) - def getFetcherLagMetrics(topic: String, partitionId: Int): FetcherLagMetrics = { - stats.getAndMaybePut( (topic, partitionId) ) + def getFetcherLagStats(topic: String, partitionId: Int): FetcherLagMetrics = { + stats.getAndMaybePut(new ClientIdTopicPartition(clientId, topic, partitionId)) } } -class FetcherStat(name: String) extends KafkaMetricsGroup { - val requestRate = newMeter(name + "RequestsPerSec", "requests", TimeUnit.SECONDS) - val byteRate = newMeter(name + "BytesPerSec", "bytes", TimeUnit.SECONDS) +class FetcherMetrics(clientIdTopic: ClientIdAndTopic) extends KafkaMetricsGroup { + val requestRate = newMeter(clientIdTopic + "-RequestsPerSec", "requests", TimeUnit.SECONDS) + val byteRate = newMeter(clientIdTopic + "-BytesPerSec", "bytes", TimeUnit.SECONDS) } -object FetcherStat { - private val valueFactory = (k: String) => new FetcherStat(k) - private val stats = new Pool[String, FetcherStat](Some(valueFactory)) +class FetcherStats(clientId: String) { + private val valueFactory = (k: ClientIdAndTopic) => new FetcherMetrics(k) + private val stats = new Pool[ClientIdAndTopic, FetcherMetrics](Some(valueFactory)) - def getFetcherStat(name: String): FetcherStat = { - stats.getAndMaybePut(name) + def getFetcherStats(name: String): FetcherMetrics = { + stats.getAndMaybePut(new ClientIdAndTopic(clientId, name)) } } + +case class ClientIdTopicPartition(clientId: String, topic: String, partitionId: Int) { + override def toString = "%s-%s-%d".format(clientId, topic, partitionId) +} diff --git a/core/src/main/scala/kafka/server/HighwaterMarkCheckpoint.scala b/core/src/main/scala/kafka/server/HighwaterMarkCheckpoint.scala index c2d953f..b726a7e 100644 --- a/core/src/main/scala/kafka/server/HighwaterMarkCheckpoint.scala +++ b/core/src/main/scala/kafka/server/HighwaterMarkCheckpoint.scala @@ -30,7 +30,7 @@ import java.io._ */ object HighwaterMarkCheckpoint { - val highWatermarkFileName = ".highwatermark" + val highWatermarkFileName = "replication-offset-checkpoint" val currentHighwaterMarkFileVersion = 0 } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index eff627c..3b88f8b 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -71,16 +71,16 @@ class KafkaApis(val requestChannel: RequestChannel, case (topicAndPartition, data) => (topicAndPartition, ProducerResponseStatus(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1l)) } - val errorResponse = ProducerResponse(apiRequest.versionId, apiRequest.correlationId, producerResponseStatus) + val errorResponse = ProducerResponse(apiRequest.correlationId, producerResponseStatus) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) error("error when handling request %s".format(apiRequest), e) case RequestKeys.FetchKey => val apiRequest = request.requestObj.asInstanceOf[FetchRequest] val fetchResponsePartitionData = apiRequest.requestInfo.map { case (topicAndPartition, data) => - (topicAndPartition, FetchResponsePartitionData(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), 0, -1, null)) + (topicAndPartition, FetchResponsePartitionData(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1, null)) } - val errorResponse = FetchResponse(apiRequest.versionId, apiRequest.correlationId, fetchResponsePartitionData) + val errorResponse = FetchResponse(apiRequest.correlationId, fetchResponsePartitionData) requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(errorResponse))) error("error when handling request %s".format(apiRequest), e) case RequestKeys.OffsetsKey => @@ -89,7 +89,7 @@ class KafkaApis(val requestChannel: RequestChannel, case (topicAndPartition, partitionOffsetRequest) => (topicAndPartition, PartitionOffsetsResponse(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), null)) } - val errorResponse = OffsetResponse(apiRequest.versionId, partitionOffsetResponseMap) + val errorResponse = OffsetResponse(apiRequest.correlationId, partitionOffsetResponseMap) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) error("error when handling request %s".format(apiRequest), e) case RequestKeys.MetadataKey => @@ -97,7 +97,7 @@ class KafkaApis(val requestChannel: RequestChannel, val topicMeatadata = apiRequest.topics.map { topic => TopicMetadata(topic, Nil, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } - val errorResponse = TopicMetadataResponse(apiRequest.versionId, topicMeatadata) + val errorResponse = TopicMetadataResponse(topicMeatadata, apiRequest.correlationId) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) error("error when handling request %s".format(apiRequest), e) case RequestKeys.LeaderAndIsrKey => @@ -105,7 +105,7 @@ class KafkaApis(val requestChannel: RequestChannel, val responseMap = apiRequest.partitionStateInfos.map { case (topicAndPartition, partitionAndState) => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } - val errorResponse = LeaderAndIsrResponse(apiRequest.versionId, responseMap) + val errorResponse = LeaderAndIsrResponse(apiRequest.correlationId, responseMap) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) error("error when handling request %s".format(apiRequest), e) case RequestKeys.StopReplicaKey => @@ -114,7 +114,7 @@ class KafkaApis(val requestChannel: RequestChannel, case topicAndPartition => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) }.toMap error("error when handling request %s".format(apiRequest), e) - val errorResponse = StopReplicaResponse(apiRequest.versionId, responseMap) + val errorResponse = StopReplicaResponse(apiRequest.correlationId, responseMap) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) } } finally @@ -128,7 +128,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Handling leader and ISR request " + leaderAndIsrRequest) try { val (response, error) = replicaManager.becomeLeaderOrFollower(leaderAndIsrRequest) - val leaderAndIsrResponse = new LeaderAndIsrResponse(leaderAndIsrRequest.versionId, response, error) + val leaderAndIsrResponse = new LeaderAndIsrResponse(leaderAndIsrRequest.correlationId, response, error) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(leaderAndIsrResponse))) } catch { case e: KafkaStorageException => @@ -145,7 +145,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Handling stop replica request " + stopReplicaRequest) val (response, error) = replicaManager.stopReplicas(stopReplicaRequest) - val stopReplicaResponse = new StopReplicaResponse(stopReplicaRequest.versionId, response.toMap, error) + val stopReplicaResponse = new StopReplicaResponse(stopReplicaRequest.correlationId, response.toMap, error) requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(stopReplicaResponse))) replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads() @@ -162,7 +162,7 @@ class KafkaApis(val requestChannel: RequestChannel, // send any newly unblocked responses for(fetchReq <- satisfied) { val topicData = readMessageSets(fetchReq.fetch) - val response = FetchResponse(FetchRequest.CurrentVersion, fetchReq.fetch.correlationId, topicData) + val response = FetchResponse(fetchReq.fetch.correlationId, topicData) requestChannel.sendResponse(new RequestChannel.Response(fetchReq.request, new FetchResponseSend(response))) } } @@ -193,7 +193,7 @@ class KafkaApis(val requestChannel: RequestChannel, allPartitionHaveReplicationFactorOne || numPartitionsInError == produceRequest.numPartitions) { val statuses = localProduceResults.map(r => r.key -> ProducerResponseStatus(r.errorCode, r.start)).toMap - val response = ProducerResponse(produceRequest.versionId, produceRequest.correlationId, statuses) + val response = ProducerResponse(produceRequest.correlationId, statuses) requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) } else { // create a list of (topic, partition) pairs to use as keys for this delayed request @@ -238,18 +238,14 @@ class KafkaApis(val requestChannel: RequestChannel, private def appendToLocalLog(partitionAndData: Map[TopicAndPartition, MessageSet]): Iterable[ProduceResult] = { trace("Append [%s] to local log ".format(partitionAndData.toString)) partitionAndData.map {case (topicAndPartition, messages) => - BrokerTopicStat.getBrokerTopicStat(topicAndPartition.topic).bytesInRate.mark(messages.sizeInBytes) - BrokerTopicStat.getBrokerAllTopicStat.bytesInRate.mark(messages.sizeInBytes) + BrokerTopicStats.getBrokerTopicStats(topicAndPartition.topic).bytesInRate.mark(messages.sizeInBytes) + BrokerTopicStats.getBrokerAllTopicStats.bytesInRate.mark(messages.sizeInBytes) try { val localReplica = replicaManager.getLeaderReplicaIfLocal(topicAndPartition.topic, topicAndPartition.partition) val log = localReplica.log.get val info = log.append(messages.asInstanceOf[ByteBufferMessageSet], assignOffsets = true) - // update stats - BrokerTopicStat.getBrokerTopicStat(topicAndPartition.topic).messagesInRate.mark(info.count) - BrokerTopicStat.getBrokerAllTopicStat.messagesInRate.mark(info.count) - // we may need to increment high watermark since ISR could be down to 1 localReplica.partition.maybeIncrementLeaderHW(localReplica) trace("%d bytes written to log %s-%d beginning at offset %d and ending at offset %d" @@ -261,8 +257,8 @@ class KafkaApis(val requestChannel: RequestChannel, Runtime.getRuntime.halt(1) null case e => - BrokerTopicStat.getBrokerTopicStat(topicAndPartition.topic).failedProduceRequestRate.mark() - BrokerTopicStat.getBrokerAllTopicStat.failedProduceRequestRate.mark() + BrokerTopicStats.getBrokerTopicStats(topicAndPartition.topic).failedProduceRequestRate.mark() + BrokerTopicStats.getBrokerAllTopicStats.failedProduceRequestRate.mark() error("Error processing ProducerRequest on %s:%d".format(topicAndPartition.topic, topicAndPartition.partition), e) new ProduceResult(topicAndPartition, e) } @@ -298,7 +294,7 @@ class KafkaApis(val requestChannel: RequestChannel, bytesReadable >= fetchRequest.minBytes || fetchRequest.numPartitions <= 0) { debug("Returning fetch response %s for fetch request with correlation id %d".format(dataRead.values.map(_.error).mkString(","), fetchRequest.correlationId)) - val response = new FetchResponse(FetchRequest.CurrentVersion, fetchRequest.correlationId, dataRead) + val response = new FetchResponse(fetchRequest.correlationId, dataRead) requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(response))) } else { debug("Putting fetch request into purgatory") @@ -329,22 +325,21 @@ class KafkaApis(val requestChannel: RequestChannel, val partitionData = try { val (messages, highWatermark) = readMessageSet(topic, partition, offset, fetchSize, fetchRequest.replicaId) - BrokerTopicStat.getBrokerTopicStat(topic).bytesOutRate.mark(messages.sizeInBytes) - BrokerTopicStat.getBrokerAllTopicStat.bytesOutRate.mark(messages.sizeInBytes) + BrokerTopicStats.getBrokerTopicStats(topic).bytesOutRate.mark(messages.sizeInBytes) + BrokerTopicStats.getBrokerAllTopicStats.bytesOutRate.mark(messages.sizeInBytes) if (!isFetchFromFollower) { - new FetchResponsePartitionData(ErrorMapping.NoError, offset, highWatermark, messages) + new FetchResponsePartitionData(ErrorMapping.NoError, highWatermark, messages) } else { debug("Leader %d for topic %s partition %d received fetch request from follower %d" .format(brokerId, topic, partition, fetchRequest.replicaId)) - new FetchResponsePartitionData(ErrorMapping.NoError, offset, highWatermark, messages) + new FetchResponsePartitionData(ErrorMapping.NoError, highWatermark, messages) } } catch { case t: Throwable => - BrokerTopicStat.getBrokerTopicStat(topic).failedFetchRequestRate.mark() - BrokerTopicStat.getBrokerAllTopicStat.failedFetchRequestRate.mark() + BrokerTopicStats.getBrokerTopicStats(topic).failedFetchRequestRate.mark() + BrokerTopicStats.getBrokerAllTopicStats.failedFetchRequestRate.mark() error("error when processing request " + (topic, partition, offset, fetchSize), t) - new FetchResponsePartitionData(ErrorMapping.codeFor(t.getClass.asInstanceOf[Class[Throwable]]), - offset, -1L, MessageSet.Empty) + new FetchResponsePartitionData(ErrorMapping.codeFor(t.getClass.asInstanceOf[Class[Throwable]]), -1L, MessageSet.Empty) } (TopicAndPartition(topic, partition), partitionData) } @@ -418,7 +413,7 @@ class KafkaApis(val requestChannel: RequestChannel, (topicAndPartition, PartitionOffsetsResponse(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), Nil) ) } }) - val response = OffsetResponse(OffsetRequest.CurrentVersion, responseMap) + val response = OffsetResponse(offsetRequest.correlationId, responseMap) requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) } @@ -496,9 +491,13 @@ class KafkaApis(val requestChannel: RequestChannel, try { /* check if auto creation of topics is turned on */ if (config.autoCreateTopics) { - CreateTopicCommand.createTopic(zkClient, topicAndMetadata.topic, config.numPartitions, config.defaultReplicationFactor) - info("Auto creation of topic %s with %d partitions and replication factor %d is successful!" - .format(topicAndMetadata.topic, config.numPartitions, config.defaultReplicationFactor)) + try { + CreateTopicCommand.createTopic(zkClient, topicAndMetadata.topic, config.numPartitions, config.defaultReplicationFactor) + info("Auto creation of topic %s with %d partitions and replication factor %d is successful!" + .format(topicAndMetadata.topic, config.numPartitions, config.defaultReplicationFactor)) + } catch { + case e: TopicExistsException => // let it go, possibly another broker created this topic + } val newTopicMetadata = AdminUtils.fetchTopicMetadataFromZk(topicAndMetadata.topic, zkClient) topicsMetadata += newTopicMetadata newTopicMetadata.errorCode match { @@ -516,7 +515,7 @@ class KafkaApis(val requestChannel: RequestChannel, } }) topicsMetadata.foreach(metadata => trace("Sending topic metadata " + metadata.toString)) - val response = new TopicMetadataResponse(metadataRequest.versionId, topicsMetadata.toSeq) + val response = new TopicMetadataResponse(topicsMetadata.toSeq, metadataRequest.correlationId) requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) } @@ -573,7 +572,7 @@ class KafkaApis(val requestChannel: RequestChannel, debug("Expiring fetch request %s.".format(delayed.fetch)) try { val topicData = readMessageSets(delayed.fetch) - val response = FetchResponse(FetchRequest.CurrentVersion, delayed.fetch.correlationId, topicData) + val response = FetchResponse(delayed.fetch.correlationId, topicData) val fromFollower = delayed.fetch.isFromFollower delayedRequestMetrics.recordDelayedFetchExpired(fromFollower) requestChannel.sendResponse(new RequestChannel.Response(delayed.request, new FetchResponseSend(response))) @@ -623,7 +622,7 @@ class KafkaApis(val requestChannel: RequestChannel, (status._1, ProducerResponseStatus(pstat.error, pstat.requiredOffset)) }) - val response = ProducerResponse(produce.versionId, produce.correlationId, finalErrorsAndOffsets) + val response = ProducerResponse(produce.correlationId, finalErrorsAndOffsets) requestChannel.sendResponse(new RequestChannel.Response( request, new BoundedByteBufferSend(response))) diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala index d949652..f0c05a5 100644 --- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala +++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala @@ -79,14 +79,14 @@ class BrokerTopicMetrics(name: String) extends KafkaMetricsGroup { val failedFetchRequestRate = newMeter(name + "FailedFetchRequestsPerSec", "requests", TimeUnit.SECONDS) } -object BrokerTopicStat extends Logging { +object BrokerTopicStats extends Logging { private val valueFactory = (k: String) => new BrokerTopicMetrics(k) private val stats = new Pool[String, BrokerTopicMetrics](Some(valueFactory)) - private val allTopicStat = new BrokerTopicMetrics("AllTopics") + private val allTopicStats = new BrokerTopicMetrics("AllTopics") - def getBrokerAllTopicStat(): BrokerTopicMetrics = allTopicStat + def getBrokerAllTopicStats(): BrokerTopicMetrics = allTopicStats - def getBrokerTopicStat(topic: String): BrokerTopicMetrics = { + def getBrokerTopicStats(topic: String): BrokerTopicMetrics = { stats.getAndMaybePut(topic + "-") } } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 725226a..36bd27d 100644 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -23,7 +23,7 @@ import kafka.utils._ import java.util.concurrent._ import atomic.AtomicBoolean import org.I0Itec.zkclient.ZkClient -import kafka.controller.{ControllerStat, KafkaController} +import kafka.controller.{ControllerStats, KafkaController} /** * Represents the lifecycle of a single Kafka broker. Handles all functionality required @@ -96,9 +96,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg * Forces some dynamic jmx beans to be registered on server startup. */ private def registerStats() { - BrokerTopicStat.getBrokerAllTopicStat() - ControllerStat.offlinePartitionRate - ControllerStat.uncleanLeaderElectionRate + BrokerTopicStats.getBrokerAllTopicStats() + ControllerStats.offlinePartitionRate + ControllerStats.uncleanLeaderElectionRate } /** diff --git a/core/src/main/scala/kafka/server/KafkaZooKeeper.scala b/core/src/main/scala/kafka/server/KafkaZooKeeper.scala index 67a0be8..e1c11f2 100644 --- a/core/src/main/scala/kafka/server/KafkaZooKeeper.scala +++ b/core/src/main/scala/kafka/server/KafkaZooKeeper.scala @@ -43,8 +43,7 @@ class KafkaZooKeeper(config: KafkaConfig) extends Logging { private def registerBrokerInZk() { info("Registering broker " + brokerIdPath) val hostName = config.hostName - val creatorId = hostName + "-" + System.currentTimeMillis - ZkUtils.registerBrokerInZk(zkClient, config.brokerId, hostName, creatorId, config.port) + ZkUtils.registerBrokerInZk(zkClient, config.brokerId, hostName, config.port) } /** diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala index 8dc2b2a..c5359b0 100644 --- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala @@ -28,7 +28,7 @@ class ReplicaFetcherThread(name:String, brokerConfig: KafkaConfig, replicaMgr: ReplicaManager) extends AbstractFetcherThread(name = name, - clientId = FetchRequest.ReplicaFetcherClientId + "- %s:%d".format(sourceBroker.host, sourceBroker.port) , + clientId = FetchRequest.ReplicaFetcherClientId, sourceBroker = sourceBroker, socketTimeout = brokerConfig.replicaSocketTimeoutMs, socketBufferSize = brokerConfig.replicaSocketBufferSize, diff --git a/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala b/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala index 9ca4dc8..db9acc9 100644 --- a/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala +++ b/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala @@ -41,7 +41,7 @@ object ConsumerOffsetChecker extends Logging { val brokerInfo = ZkUtils.readDataMaybeNull(zkClient, "/brokers/ids/%s".format(bid))._1 val consumer = brokerInfo match { case BrokerIpPattern(ip, port) => - Some(new SimpleConsumer(ip, port.toInt, 10000, 100000)) + Some(new SimpleConsumer(ip, port.toInt, 10000, 100000, "ConsumerOffsetChecker")) case _ => error("Could not parse broker info %s".format(brokerInfo)) None diff --git a/core/src/main/scala/kafka/tools/DumpLogSegments.scala b/core/src/main/scala/kafka/tools/DumpLogSegments.scala index 167cf10..31333e7 100644 --- a/core/src/main/scala/kafka/tools/DumpLogSegments.scala +++ b/core/src/main/scala/kafka/tools/DumpLogSegments.scala @@ -57,8 +57,8 @@ object DumpLogSegments { val verifyOnly = if(options.has(verifyOpt)) true else false val files = options.valueOf(filesOpt).split(",") - val misMatchesForIndexFilesMap = new mutable.HashMap[String, List[(Int, Int)]] - val nonConsecutivePairsForLogFilesMap = new mutable.HashMap[String, List[(Int, Int)]] + val misMatchesForIndexFilesMap = new mutable.HashMap[String, List[(Long, Long)]] + val nonConsecutivePairsForLogFilesMap = new mutable.HashMap[String, List[(Long, Long)]] for(arg <- files) { val file = new File(arg) @@ -89,7 +89,7 @@ object DumpLogSegments { } /* print out the contents of the index */ - private def dumpIndex(file: File, verifyOnly: Boolean, misMatchesForIndexFilesMap: mutable.HashMap[String, List[(Int, Int)]]) { + private def dumpIndex(file: File, verifyOnly: Boolean, misMatchesForIndexFilesMap: mutable.HashMap[String, List[(Long, Long)]]) { val startOffset = file.getName().split("\\.")(0).toLong val logFileName = file.getAbsolutePath.split("\\.")(0) + Log.LogFileSuffix val logFile = new File(logFileName) @@ -100,8 +100,8 @@ object DumpLogSegments { val partialFileMessageSet: FileMessageSet = messageSet.read(entry.position, messageSet.sizeInBytes()) val messageAndOffset = partialFileMessageSet.head if(messageAndOffset.offset != entry.offset + index.baseOffset) { - var misMatchesSeq = misMatchesForIndexFilesMap.getOrElse(file.getName, List[(Int, Int)]()) - misMatchesSeq ::=((entry.offset + index.baseOffset, messageAndOffset.offset).asInstanceOf[(Int, Int)]) + var misMatchesSeq = misMatchesForIndexFilesMap.getOrElse(file.getName, List[(Long, Long)]()) + misMatchesSeq ::=(entry.offset + index.baseOffset, messageAndOffset.offset) misMatchesForIndexFilesMap.put(file.getName, misMatchesSeq) } // since it is a sparse file, in the event of a crash there may be many zero entries, stop if we see one @@ -113,7 +113,7 @@ object DumpLogSegments { } /* print out the contents of the log */ - private def dumpLog(file: File, printContents: Boolean, nonConsecutivePairsForLogFilesMap: mutable.HashMap[String, List[(Int, Int)]]) { + private def dumpLog(file: File, printContents: Boolean, nonConsecutivePairsForLogFilesMap: mutable.HashMap[String, List[(Long, Long)]]) { val startOffset = file.getName().split("\\.")(0).toLong println("Starting offset: " + startOffset) val messageSet = new FileMessageSet(file) @@ -126,8 +126,8 @@ object DumpLogSegments { lastOffset = messageAndOffset.offset // If it's uncompressed message, its offset must be lastOffset + 1 no matter last message is compressed or uncompressed else if (msg.compressionCodec == NoCompressionCodec && messageAndOffset.offset != lastOffset +1) { - var nonConsecutivePairsSeq = nonConsecutivePairsForLogFilesMap.getOrElse(file.getName, List[(Int, Int)]()) - nonConsecutivePairsSeq ::=((lastOffset, messageAndOffset.offset).asInstanceOf[(Int, Int)]) + var nonConsecutivePairsSeq = nonConsecutivePairsForLogFilesMap.getOrElse(file.getName, List[(Long, Long)]()) + nonConsecutivePairsSeq ::=(lastOffset, messageAndOffset.offset) nonConsecutivePairsForLogFilesMap.put(file.getName, nonConsecutivePairsSeq) } lastOffset = messageAndOffset.offset diff --git a/core/src/main/scala/kafka/tools/GetOffsetShell.scala b/core/src/main/scala/kafka/tools/GetOffsetShell.scala index e78d53d..2b9438a 100644 --- a/core/src/main/scala/kafka/tools/GetOffsetShell.scala +++ b/core/src/main/scala/kafka/tools/GetOffsetShell.scala @@ -67,7 +67,7 @@ object GetOffsetShell { val partition = options.valueOf(partitionOpt).intValue var time = options.valueOf(timeOpt).longValue val nOffsets = options.valueOf(nOffsetsOpt).intValue - val consumer = new SimpleConsumer(url.getHost, url.getPort, 10000, 100000) + val consumer = new SimpleConsumer(url.getHost, url.getPort, 10000, 100000, "GetOffsetShell") val topicAndPartition = TopicAndPartition(topic, partition) val request = OffsetRequest(Map(topicAndPartition -> PartitionOffsetRequestInfo(time, nOffsets))) val offsets = consumer.getOffsetsBefore(request).partitionErrorAndOffsets(topicAndPartition).offsets diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala index aa23559..5c4b3d2 100644 --- a/core/src/main/scala/kafka/tools/MirrorMaker.scala +++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala @@ -17,7 +17,6 @@ package kafka.tools -import kafka.message.Message import joptsimple.OptionParser import kafka.utils.{Utils, CommandLineUtils, Logging} import kafka.producer.{KeyedMessage, ProducerConfig, Producer} diff --git a/core/src/main/scala/kafka/tools/ProducerShell.scala b/core/src/main/scala/kafka/tools/ProducerShell.scala deleted file mode 100644 index 09b279f..0000000 --- a/core/src/main/scala/kafka/tools/ProducerShell.scala +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.tools - -import java.io._ -import joptsimple._ -import kafka.producer._ -import kafka.utils.Utils - -/** - * Interactive shell for producing messages from the command line - */ -object ProducerShell { - - def main(args: Array[String]) { - - val parser = new OptionParser - val producerPropsOpt = parser.accepts("props", "REQUIRED: Properties file with the producer properties.") - .withRequiredArg - .describedAs("properties") - .ofType(classOf[String]) - val topicOpt = parser.accepts("topic", "REQUIRED: The topic to produce to.") - .withRequiredArg - .describedAs("topic") - .ofType(classOf[String]) - - val options = parser.parse(args : _*) - - for(arg <- List(producerPropsOpt, topicOpt)) { - if(!options.has(arg)) { - System.err.println("Missing required argument \"" + arg + "\"") - parser.printHelpOn(System.err) - System.exit(1) - } - } - - val propsFile = options.valueOf(producerPropsOpt) - val producerConfig = new ProducerConfig(Utils.loadProps(propsFile)) - val topic = options.valueOf(topicOpt) - val producer = new Producer[String, String](producerConfig) - - val input = new BufferedReader(new InputStreamReader(System.in)) - var done = false - while(!done) { - val line = input.readLine() - if(line == null) { - done = true - } else { - val message = line.trim - producer.send(new KeyedMessage[String, String](topic, message)) - println("Sent: %s (%d bytes)".format(line, message.getBytes.length)) - } - } - producer.close() - } -} diff --git a/core/src/main/scala/kafka/tools/ReplayLogProducer.scala b/core/src/main/scala/kafka/tools/ReplayLogProducer.scala index 79ffcc5..db14c82 100644 --- a/core/src/main/scala/kafka/tools/ReplayLogProducer.scala +++ b/core/src/main/scala/kafka/tools/ReplayLogProducer.scala @@ -24,7 +24,7 @@ import kafka.producer.{KeyedMessage, ProducerConfig, Producer} import kafka.consumer._ import kafka.utils.{Logging, ZkUtils} import kafka.api.OffsetRequest -import kafka.message.{CompressionCodec, Message} +import kafka.message.CompressionCodec object ReplayLogProducer extends Logging { diff --git a/core/src/main/scala/kafka/tools/SimpleConsumerShell.scala b/core/src/main/scala/kafka/tools/SimpleConsumerShell.scala index dac7056..1842c03 100644 --- a/core/src/main/scala/kafka/tools/SimpleConsumerShell.scala +++ b/core/src/main/scala/kafka/tools/SimpleConsumerShell.scala @@ -19,12 +19,10 @@ package kafka.tools import joptsimple._ import kafka.utils._ -import kafka.producer.ProducerConfig import kafka.consumer._ import kafka.client.ClientUtils import kafka.api.{OffsetRequest, FetchRequestBuilder, Request} import kafka.cluster.Broker -import java.util.Properties import scala.collection.JavaConversions._ /** @@ -74,7 +72,7 @@ object SimpleConsumerShell extends Logging { .withRequiredArg .describedAs("class") .ofType(classOf[String]) - .defaultsTo(classOf[NewlineMessageFormatter].getName) + .defaultsTo(classOf[DefaultMessageFormatter].getName) val messageFormatterArgOpt = parser.accepts("property") .withRequiredArg .describedAs("prop") @@ -127,7 +125,7 @@ object SimpleConsumerShell extends Logging { // getting topic metadata info("Getting topic metatdata...") val metadataTargetBrokers = ClientUtils.parseBrokerList(options.valueOf(brokerListOpt)) - val topicsMetadata = ClientUtils.fetchTopicMetadata(Set(topic), metadataTargetBrokers).topicsMetadata + val topicsMetadata = ClientUtils.fetchTopicMetadata(Set(topic), metadataTargetBrokers, clientId).topicsMetadata if(topicsMetadata.size != 1 || !topicsMetadata(0).topic.equals(topic)) { System.err.println(("Error: no valid topic metadata for topic: %s, " + "what we get from server is only: %s").format(topic, topicsMetadata)) System.exit(1) @@ -167,7 +165,7 @@ object SimpleConsumerShell extends Logging { System.exit(1) } if(startingOffset < 0) - startingOffset = SimpleConsumer.earliestOrLatestOffset(fetchTargetBroker, topic, partitionId, startingOffset, false) + startingOffset = SimpleConsumer.earliestOrLatestOffset(fetchTargetBroker, topic, partitionId, startingOffset, clientId, false) // initializing formatter val formatter: MessageFormatter = messageFormatterClass.newInstance().asInstanceOf[MessageFormatter] @@ -175,7 +173,7 @@ object SimpleConsumerShell extends Logging { info("Starting simple consumer shell to partition [%s, %d], replica [%d], host and port: [%s, %d], from offset [%d]" .format(topic, partitionId, replicaId, fetchTargetBroker.host, fetchTargetBroker.port, startingOffset)) - val simpleConsumer = new SimpleConsumer(fetchTargetBroker.host, fetchTargetBroker.port, 10000, 64*1024) + val simpleConsumer = new SimpleConsumer(fetchTargetBroker.host, fetchTargetBroker.port, 10000, 64*1024, clientId) val thread = Utils.newThread("kafka-simpleconsumer-shell", new Runnable() { def run() { var offset = startingOffset diff --git a/core/src/main/scala/kafka/tools/UpdateOffsetsInZK.scala b/core/src/main/scala/kafka/tools/UpdateOffsetsInZK.scala index fed7aad..111c9a8 100644 --- a/core/src/main/scala/kafka/tools/UpdateOffsetsInZK.scala +++ b/core/src/main/scala/kafka/tools/UpdateOffsetsInZK.scala @@ -65,7 +65,7 @@ object UpdateOffsetsInZK { ZkUtils.getBrokerInfo(zkClient, broker) match { case Some(brokerInfo) => - val consumer = new SimpleConsumer(brokerInfo.host, brokerInfo.port, 10000, 100 * 1024) + val consumer = new SimpleConsumer(brokerInfo.host, brokerInfo.port, 10000, 100 * 1024, "UpdateOffsetsInZk") val topicAndPartition = TopicAndPartition(topic, partition) val request = OffsetRequest(Map(topicAndPartition -> PartitionOffsetRequestInfo(offsetOption, 1))) val offset = consumer.getOffsetsBefore(request).partitionErrorAndOffsets(topicAndPartition).offsets.head diff --git a/core/src/main/scala/kafka/utils/ClientIdAndTopic.scala b/core/src/main/scala/kafka/utils/ClientIdAndTopic.scala new file mode 100644 index 0000000..780339e --- /dev/null +++ b/core/src/main/scala/kafka/utils/ClientIdAndTopic.scala @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.utils + +import kafka.common.InvalidTopicException +import kafka.common.InvalidClientIdException +import util.matching.Regex + +object ClientId { + val legalChars = "[a-zA-Z0-9_-]" + val maxNameLength = 200 // to prevent hitting filename max length limit + private val rgx = new Regex(legalChars + "*") + + def validate(clientId: String) { + if (clientId.length > maxNameLength) + throw new InvalidClientIdException("ClientId is illegal, can't be longer than " + maxNameLength + " characters") + + rgx.findFirstIn(clientId) match { + case Some(t) => + if (!t.equals(clientId)) + throw new InvalidClientIdException("ClientId " + clientId + " is illegal, contains a character other than ASCII alphanumerics, _ and -") + case None => throw new InvalidClientIdException("ClientId " + clientId + " is illegal, contains a character other than ASCII alphanumerics, _ and -") + } + } +} + +object Topic { + val legalChars = "[a-zA-Z0-9_-]" + val maxNameLength = 255 + private val rgx = new Regex(legalChars + "+") + + def validate(topic: String) { + if (topic.length <= 0) + throw new InvalidTopicException("topic name is illegal, can't be empty") + else if (topic.length > maxNameLength) + throw new InvalidTopicException("topic name is illegal, can't be longer than " + maxNameLength + " characters") + + rgx.findFirstIn(topic) match { + case Some(t) => + if (!t.equals(topic)) + throw new InvalidTopicException("topic name " + topic + " is illegal, contains a character other than ASCII alphanumerics, _ and -") + case None => throw new InvalidTopicException("topic name " + topic + " is illegal, contains a character other than ASCII alphanumerics, _ and -") + } + } +} + +case class ClientIdAndTopic(clientId: String, topic:String) { + override def toString = "%s-%s".format(clientId, topic) +} \ No newline at end of file diff --git a/core/src/main/scala/kafka/utils/Topic.scala b/core/src/main/scala/kafka/utils/Topic.scala index fe79adf..e69de29 100644 --- a/core/src/main/scala/kafka/utils/Topic.scala +++ b/core/src/main/scala/kafka/utils/Topic.scala @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.utils - -import kafka.common.InvalidTopicException -import util.matching.Regex - -object Topic { - val legalChars = "[a-zA-Z0-9_-]" - val maxNameLength = 255 - private val rgx = new Regex(legalChars + "+") - - def validate(topic: String) { - if (topic.length <= 0) - throw new InvalidTopicException("topic name is illegal, can't be empty") - else if (topic.length > maxNameLength) - throw new InvalidTopicException("topic name is illegal, can't be longer than " + maxNameLength + " characters") - - rgx.findFirstIn(topic) match { - case Some(t) => - if (!t.equals(topic)) - throw new InvalidTopicException("topic name " + topic + " is illegal, contains a character other than ASCII alphanumerics, _ and -") - case None => throw new InvalidTopicException("topic name " + topic + " is illegal, contains a character other than ASCII alphanumerics, _ and -") - } - } -} diff --git a/core/src/main/scala/kafka/utils/Utils.scala b/core/src/main/scala/kafka/utils/Utils.scala index d383f54..1640428 100644 --- a/core/src/main/scala/kafka/utils/Utils.scala +++ b/core/src/main/scala/kafka/utils/Utils.scala @@ -406,7 +406,7 @@ object Utils extends Logging { } /** - * This method gets comma seperated values which contains key,value pairs and returns a map of + * This method gets comma separated values which contains key,value pairs and returns a map of * key value pairs. the format of allCSVal is key1:val1, key2:val2 .... */ def parseCsvMap(str: String): Map[String, String] = { diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 4333401..358c4fd 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -180,11 +180,11 @@ object ZkUtils extends Logging { replicas.contains(brokerId.toString) } - def registerBrokerInZk(zkClient: ZkClient, id: Int, host: String, creator: String, port: Int) { + def registerBrokerInZk(zkClient: ZkClient, id: Int, host: String, port: Int) { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + id - val broker = new Broker(id, creator, host, port) + val broker = new Broker(id, host, port) try { - createEphemeralPathExpectConflict(zkClient, brokerIdPath, broker.getZKString) + createEphemeralPathExpectConflict(zkClient, brokerIdPath, broker.getZkString) } catch { case e: ZkNodeExistsException => throw new RuntimeException("A broker is already registered on the path " + brokerIdPath + ". This probably " + "indicates that you either have configured a brokerid that is already in use, or " + "else you have shutdown this broker and restarted it faster than the zookeeper " + "timeout so it appears to be re-registering.") diff --git a/core/src/test/scala/other/kafka/TestKafkaAppender.scala b/core/src/test/scala/other/kafka/TestKafkaAppender.scala index bd09d78..ab807a1 100644 --- a/core/src/test/scala/other/kafka/TestKafkaAppender.scala +++ b/core/src/test/scala/other/kafka/TestKafkaAppender.scala @@ -17,7 +17,6 @@ package kafka -import message.Message import org.apache.log4j.PropertyConfigurator import kafka.utils.Logging import serializer.Encoder diff --git a/core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala b/core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala index 7d48458..5b72eed 100644 --- a/core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala +++ b/core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala @@ -18,7 +18,6 @@ package kafka import consumer._ -import message.Message import utils.Utils import java.util.concurrent.CountDownLatch diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala index 7e17da4..1671a8c 100644 --- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala @@ -22,7 +22,7 @@ import org.scalatest.junit.JUnit3Suite import kafka.zk.ZooKeeperTestHarness import kafka.server.KafkaConfig import kafka.utils.{ZkUtils, TestUtils} -import kafka.common.{ErrorMapping, TopicAndPartition} +import kafka.common.{TopicExistsException, ErrorMapping, TopicAndPartition} class AdminTest extends JUnit3Suite with ZooKeeperTestHarness { @@ -170,7 +170,7 @@ class AdminTest extends JUnit3Suite with ZooKeeperTestHarness { AdminUtils.createTopicPartitionAssignmentPathInZK(topic, expectedReplicaAssignment, zkClient) fail("shouldn't be able to create a topic already exists") } catch { - case e: AdministrationException => // this is good + case e: TopicExistsException => // this is good case e2 => throw e2 } } @@ -374,33 +374,37 @@ class AdminTest extends JUnit3Suite with ZooKeeperTestHarness { var controllerId = ZkUtils.getController(zkClient) var controller = servers.find(p => p.config.brokerId == controllerId).get.kafkaController var partitionsRemaining = controller.shutdownBroker(2) - assertEquals(0, partitionsRemaining) - var topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) - var leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id - assertTrue(leaderAfterShutdown != leaderBeforeShutdown) - assertEquals(2, topicMetadata.partitionsMetadata.head.isr.size) - - leaderBeforeShutdown = leaderAfterShutdown - controllerId = ZkUtils.getController(zkClient) - controller = servers.find(p => p.config.brokerId == controllerId).get.kafkaController - partitionsRemaining = controller.shutdownBroker(1) - assertEquals(0, partitionsRemaining) - topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) - leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id - assertTrue(leaderAfterShutdown != leaderBeforeShutdown) - assertEquals(1, topicMetadata.partitionsMetadata.head.isr.size) + try { + assertEquals(0, partitionsRemaining) + var topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) + var leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id + assertTrue(leaderAfterShutdown != leaderBeforeShutdown) + assertEquals(2, controller.controllerContext.allLeaders(TopicAndPartition("test", 1)).leaderAndIsr.isr.size) - leaderBeforeShutdown = leaderAfterShutdown - controllerId = ZkUtils.getController(zkClient) - controller = servers.find(p => p.config.brokerId == controllerId).get.kafkaController - partitionsRemaining = controller.shutdownBroker(0) - assertEquals(1, partitionsRemaining) - topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) - leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id - assertTrue(leaderAfterShutdown == leaderBeforeShutdown) - assertEquals(1, topicMetadata.partitionsMetadata.head.isr.size) + leaderBeforeShutdown = leaderAfterShutdown + controllerId = ZkUtils.getController(zkClient) + controller = servers.find(p => p.config.brokerId == controllerId).get.kafkaController + partitionsRemaining = controller.shutdownBroker(1) + assertEquals(0, partitionsRemaining) + topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) + leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id + assertTrue(leaderAfterShutdown != leaderBeforeShutdown) + assertEquals(1, topicMetadata.partitionsMetadata.head.isr.size) + assertEquals(1, controller.controllerContext.allLeaders(TopicAndPartition("test", 1)).leaderAndIsr.isr.size) - servers.foreach(_.shutdown()) + leaderBeforeShutdown = leaderAfterShutdown + controllerId = ZkUtils.getController(zkClient) + controller = servers.find(p => p.config.brokerId == controllerId).get.kafkaController + partitionsRemaining = controller.shutdownBroker(0) + assertEquals(1, partitionsRemaining) + topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient) + leaderAfterShutdown = topicMetadata.partitionsMetadata.head.leader.get.id + assertTrue(leaderAfterShutdown == leaderBeforeShutdown) + assertEquals(1, controller.controllerContext.allLeaders(TopicAndPartition("test", 1)).leaderAndIsr.isr.size) + assertEquals(1, topicMetadata.partitionsMetadata.head.isr.size) + } finally { + servers.foreach(_.shutdown()) + } } private def checkIfReassignPartitionPathExists(): Boolean = { diff --git a/core/src/test/scala/unit/kafka/api/RequestResponseSerializationTest.scala b/core/src/test/scala/unit/kafka/api/RequestResponseSerializationTest.scala index 531f32e..509b020 100644 --- a/core/src/test/scala/unit/kafka/api/RequestResponseSerializationTest.scala +++ b/core/src/test/scala/unit/kafka/api/RequestResponseSerializationTest.scala @@ -75,10 +75,11 @@ object SerializationTestUtils{ TopicAndPartition(topic2, 3) -> PartitionFetchInfo(4000, 100) ) - private val partitionMetaData0 = new PartitionMetadata(0, Some(new Broker(0, "creator", "localhost", 1011)), collection.immutable.Seq.empty) - private val partitionMetaData1 = new PartitionMetadata(1, Some(new Broker(0, "creator", "localhost", 1011)), collection.immutable.Seq.empty) - private val partitionMetaData2 = new PartitionMetadata(2, Some(new Broker(0, "creator", "localhost", 1011)), collection.immutable.Seq.empty) - private val partitionMetaData3 = new PartitionMetadata(3, Some(new Broker(0, "creator", "localhost", 1011)), collection.immutable.Seq.empty) + private val brokers = List(new Broker(0, "localhost", 1011), new Broker(1, "localhost", 1012), new Broker(2, "localhost", 1013)) + private val partitionMetaData0 = new PartitionMetadata(0, Some(brokers.head), replicas = brokers, isr = brokers, errorCode = 0) + private val partitionMetaData1 = new PartitionMetadata(1, Some(brokers.head), replicas = brokers, isr = brokers.tail, errorCode = 1) + private val partitionMetaData2 = new PartitionMetadata(2, Some(brokers.head), replicas = brokers, isr = brokers, errorCode = 2) + private val partitionMetaData3 = new PartitionMetadata(3, Some(brokers.head), replicas = brokers, isr = brokers.tail.tail, errorCode = 3) private val partitionMetaDataSeq = Seq(partitionMetaData0, partitionMetaData1, partitionMetaData2, partitionMetaData3) private val topicmetaData1 = new TopicMetadata(topic1, partitionMetaDataSeq) private val topicmetaData2 = new TopicMetadata(topic2, partitionMetaDataSeq) @@ -104,7 +105,7 @@ object SerializationTestUtils{ def createTestStopReplicaResponse() : StopReplicaResponse = { val responseMap = Map(((topic1, 0), ErrorMapping.NoError), ((topic2, 0), ErrorMapping.NoError)) - new StopReplicaResponse(1, responseMap.toMap) + new StopReplicaResponse(0, responseMap.toMap) } def createTestProducerRequest: ProducerRequest = { @@ -112,7 +113,7 @@ object SerializationTestUtils{ } def createTestProducerResponse: ProducerResponse = - ProducerResponse(1, 1, Map( + ProducerResponse(1, Map( TopicAndPartition(topic1, 0) -> ProducerResponseStatus(0.toShort, 10001), TopicAndPartition(topic2, 0) -> ProducerResponseStatus(0.toShort, 20001) )) @@ -122,7 +123,7 @@ object SerializationTestUtils{ } def createTestFetchResponse: FetchResponse = { - FetchResponse(1, 1, topicDataFetchResponse) + FetchResponse(1, topicDataFetchResponse) } def createTestOffsetRequest = new OffsetRequest( @@ -131,17 +132,17 @@ object SerializationTestUtils{ ) def createTestOffsetResponse: OffsetResponse = { - new OffsetResponse(OffsetRequest.CurrentVersion, collection.immutable.Map( + new OffsetResponse(0, collection.immutable.Map( TopicAndPartition(topic1, 1) -> PartitionOffsetsResponse(ErrorMapping.NoError, Seq(1000l, 2000l, 3000l, 4000l))) ) } def createTestTopicMetadataRequest: TopicMetadataRequest = { - new TopicMetadataRequest(1, "client 1", Seq(topic1, topic2)) + new TopicMetadataRequest(1, "client 1", Seq(topic1, topic2), 1) } def createTestTopicMetadataResponse: TopicMetadataResponse = { - new TopicMetadataResponse(1, Seq(topicmetaData1, topicmetaData2)) + new TopicMetadataResponse(Seq(topicmetaData1, topicmetaData2), 1) } } diff --git a/core/src/test/scala/unit/kafka/consumer/ConsumerIteratorTest.scala b/core/src/test/scala/unit/kafka/consumer/ConsumerIteratorTest.scala index 962d5f9..246b1ec 100644 --- a/core/src/test/scala/unit/kafka/consumer/ConsumerIteratorTest.scala +++ b/core/src/test/scala/unit/kafka/consumer/ConsumerIteratorTest.scala @@ -47,7 +47,7 @@ class ConsumerIteratorTest extends JUnit3Suite with KafkaServerTestHarness { val group = "group1" val consumer0 = "consumer0" val consumedOffset = 5 - val cluster = new Cluster(configs.map(c => new Broker(c.brokerId, c.brokerId.toString, "localhost", c.port))) + val cluster = new Cluster(configs.map(c => new Broker(c.brokerId, "localhost", c.port))) val queue = new LinkedBlockingQueue[FetchedDataChunk] val topicInfos = configs.map(c => new PartitionTopicInfo(topic, c.brokerId, @@ -55,7 +55,8 @@ class ConsumerIteratorTest extends JUnit3Suite with KafkaServerTestHarness { queue, new AtomicLong(consumedOffset), new AtomicLong(0), - new AtomicInteger(0))) + new AtomicInteger(0), + new ConsumerTopicStats(""))) val consumerConfig = new ConsumerConfig(TestUtils.createConsumerProperties(zkConnect, group, consumer0)) override def setUp() { @@ -78,7 +79,8 @@ class ConsumerIteratorTest extends JUnit3Suite with KafkaServerTestHarness { consumerConfig.consumerTimeoutMs, new StringDecoder(), new StringDecoder(), - enableShallowIterator = false) + enableShallowIterator = false, + consumerTopicStats = new ConsumerTopicStats("")) var receivedMessages = (0 until 5).map(i => iter.next.message).toList assertFalse(iter.hasNext) diff --git a/core/src/test/scala/unit/kafka/integration/AutoOffsetResetTest.scala b/core/src/test/scala/unit/kafka/integration/AutoOffsetResetTest.scala index 8c7f774..d7945a5 100644 --- a/core/src/test/scala/unit/kafka/integration/AutoOffsetResetTest.scala +++ b/core/src/test/scala/unit/kafka/integration/AutoOffsetResetTest.scala @@ -24,7 +24,6 @@ import kafka.server._ import org.apache.log4j.{Level, Logger} import org.scalatest.junit.JUnit3Suite import kafka.utils.TestUtils -import kafka.message.Message import kafka.serializer._ import kafka.producer.{Producer, KeyedMessage} diff --git a/core/src/test/scala/unit/kafka/integration/FetcherTest.scala b/core/src/test/scala/unit/kafka/integration/FetcherTest.scala index df754cc..021f419 100644 --- a/core/src/test/scala/unit/kafka/integration/FetcherTest.scala +++ b/core/src/test/scala/unit/kafka/integration/FetcherTest.scala @@ -23,7 +23,6 @@ import scala.collection._ import junit.framework.Assert._ import kafka.cluster._ -import kafka.message._ import kafka.server._ import org.scalatest.junit.JUnit3Suite import kafka.consumer._ @@ -41,7 +40,7 @@ class FetcherTest extends JUnit3Suite with KafkaServerTestHarness { yield new KafkaConfig(props) val messages = new mutable.HashMap[Int, Seq[Array[Byte]]] val topic = "topic" - val cluster = new Cluster(configs.map(c => new Broker(c.brokerId, c.brokerId.toString, "localhost", c.port))) + val cluster = new Cluster(configs.map(c => new Broker(c.brokerId, "localhost", c.port))) val shutdown = ZookeeperConsumerConnector.shutdownCommand val queue = new LinkedBlockingQueue[FetchedDataChunk] val topicInfos = configs.map(c => new PartitionTopicInfo(topic, @@ -50,7 +49,8 @@ class FetcherTest extends JUnit3Suite with KafkaServerTestHarness { queue, new AtomicLong(0), new AtomicLong(0), - new AtomicInteger(0))) + new AtomicInteger(0), + new ConsumerTopicStats(""))) var fetcher: ConsumerFetcherManager = null diff --git a/core/src/test/scala/unit/kafka/integration/LazyInitProducerTest.scala b/core/src/test/scala/unit/kafka/integration/LazyInitProducerTest.scala index 4411f45..c4866eb 100644 --- a/core/src/test/scala/unit/kafka/integration/LazyInitProducerTest.scala +++ b/core/src/test/scala/unit/kafka/integration/LazyInitProducerTest.scala @@ -18,7 +18,7 @@ package kafka.integration import kafka.api.FetchRequestBuilder -import kafka.message.{Message, ByteBufferMessageSet} +import kafka.message.ByteBufferMessageSet import kafka.server.{KafkaRequestHandler, KafkaConfig} import org.apache.log4j.{Level, Logger} import org.junit.Assert._ diff --git a/core/src/test/scala/unit/kafka/integration/PrimitiveApiTest.scala b/core/src/test/scala/unit/kafka/integration/PrimitiveApiTest.scala index 72902ba..402fced 100644 --- a/core/src/test/scala/unit/kafka/integration/PrimitiveApiTest.scala +++ b/core/src/test/scala/unit/kafka/integration/PrimitiveApiTest.scala @@ -25,7 +25,6 @@ import java.util.Properties import kafka.utils.Utils import kafka.producer.{KeyedMessage, Producer, ProducerConfig} import kafka.serializer._ -import kafka.message.Message import kafka.utils.TestUtils import org.apache.log4j.{Level, Logger} import org.I0Itec.zkclient.ZkClient diff --git a/core/src/test/scala/unit/kafka/integration/ProducerConsumerTestHarness.scala b/core/src/test/scala/unit/kafka/integration/ProducerConsumerTestHarness.scala index fd9fae5..caea858 100644 --- a/core/src/test/scala/unit/kafka/integration/ProducerConsumerTestHarness.scala +++ b/core/src/test/scala/unit/kafka/integration/ProducerConsumerTestHarness.scala @@ -21,7 +21,6 @@ import kafka.consumer.SimpleConsumer import org.scalatest.junit.JUnit3Suite import java.util.Properties import kafka.producer.{ProducerConfig, Producer} -import kafka.message.Message import kafka.utils.TestUtils import kafka.serializer._ @@ -44,10 +43,7 @@ trait ProducerConsumerTestHarness extends JUnit3Suite with KafkaServerTestHarnes props.put("producer.request.required.acks", "-1") props.put("serializer.class", classOf[StringEncoder].getName.toString) producer = new Producer(new ProducerConfig(props)) - consumer = new SimpleConsumer(host, - port, - 1000000, - 64*1024) + consumer = new SimpleConsumer(host, port, 1000000, 64*1024, "") } override def tearDown() { diff --git a/core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala b/core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala index 9664876..9f243f0 100644 --- a/core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala +++ b/core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala @@ -29,7 +29,7 @@ import kafka.producer.KeyedMessage import kafka.javaapi.producer.Producer import kafka.utils.IntEncoder import kafka.utils.TestUtils._ -import kafka.utils.{Utils, Logging, TestUtils} +import kafka.utils.{Logging, TestUtils} import kafka.consumer.{KafkaStream, ConsumerConfig} import kafka.zk.ZooKeeperTestHarness diff --git a/core/src/test/scala/unit/kafka/log4j/KafkaLog4jAppenderTest.scala b/core/src/test/scala/unit/kafka/log4j/KafkaLog4jAppenderTest.scala index da3c704..c25255f 100644 --- a/core/src/test/scala/unit/kafka/log4j/KafkaLog4jAppenderTest.scala +++ b/core/src/test/scala/unit/kafka/log4j/KafkaLog4jAppenderTest.scala @@ -24,7 +24,6 @@ import kafka.server.{KafkaConfig, KafkaServer} import kafka.utils.{TestUtils, Utils, Logging} import junit.framework.Assert._ import kafka.api.FetchRequestBuilder -import kafka.message.Message import kafka.producer.async.MissingConfigException import kafka.serializer.Encoder import kafka.zk.ZooKeeperTestHarness @@ -57,7 +56,7 @@ class KafkaLog4jAppenderTest extends JUnit3Suite with ZooKeeperTestHarness with logDirZk = new File(logDirZkPath) config = new KafkaConfig(propsZk) serverZk = TestUtils.createServer(config); - simpleConsumerZk = new SimpleConsumer("localhost", portZk, 1000000, 64*1024) + simpleConsumerZk = new SimpleConsumer("localhost", portZk, 1000000, 64*1024, "") } @After diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 9074ca8..3b5ec7f 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -74,7 +74,7 @@ class SocketServerTest extends JUnitSuite { @Test def simpleRequest() { val socket = connect() - val correlationId = SyncProducerConfig.DefaultCorrelationId + val correlationId = -1 val clientId = SyncProducerConfig.DefaultClientId val ackTimeoutMs = SyncProducerConfig.DefaultAckTimeoutMs val ack = SyncProducerConfig.DefaultRequiredAcks diff --git a/core/src/test/scala/unit/kafka/producer/AsyncProducerTest.scala b/core/src/test/scala/unit/kafka/producer/AsyncProducerTest.scala index 7d77eb6..2a20058 100644 --- a/core/src/test/scala/unit/kafka/producer/AsyncProducerTest.scala +++ b/core/src/test/scala/unit/kafka/producer/AsyncProducerTest.scala @@ -17,7 +17,7 @@ package kafka.producer -import java.util.{LinkedList, Properties} +import java.util.Properties import java.util.concurrent.LinkedBlockingQueue import junit.framework.Assert._ import org.easymock.EasyMock @@ -68,7 +68,10 @@ class AsyncProducerTest extends JUnit3Suite { val config = new ProducerConfig(props) val produceData = getProduceData(12) - val producer = new Producer[String, String](config, mockEventHandler) + val producer = new Producer[String, String](config, + mockEventHandler, + new ProducerStats(""), + new ProducerTopicStats("")) try { // send all 10 messages, should hit the batch size and then reach broker producer.send(produceData: _*) @@ -118,7 +121,7 @@ class AsyncProducerTest extends JUnit3Suite { val queue = new LinkedBlockingQueue[KeyedMessage[String,String]](10) val producerSendThread = - new ProducerSendThread[String,String]("thread1", queue, mockHandler, Integer.MAX_VALUE, 5) + new ProducerSendThread[String,String]("thread1", queue, mockHandler, Integer.MAX_VALUE, 5, "") producerSendThread.start() for (producerData <- producerDataList) @@ -143,7 +146,7 @@ class AsyncProducerTest extends JUnit3Suite { val queueExpirationTime = 200 val queue = new LinkedBlockingQueue[KeyedMessage[String,String]](10) val producerSendThread = - new ProducerSendThread[String,String]("thread1", queue, mockHandler, queueExpirationTime, 5) + new ProducerSendThread[String,String]("thread1", queue, mockHandler, queueExpirationTime, 5, "") producerSendThread.start() for (producerData <- producerDataList) @@ -165,8 +168,8 @@ class AsyncProducerTest extends JUnit3Suite { val props = new Properties() props.put("broker.list", TestUtils.getBrokerListStrFromConfigs(configs)) - val broker1 = new Broker(0, "localhost", "localhost", 9092) - val broker2 = new Broker(1, "localhost", "localhost", 9093) + val broker1 = new Broker(0, "localhost", 9092) + val broker2 = new Broker(1, "localhost", 9093) broker1 // form expected partitions metadata val partition1Metadata = new PartitionMetadata(0, Some(broker1), List(broker1, broker2)) @@ -185,11 +188,13 @@ class AsyncProducerTest extends JUnit3Suite { val producerPool = new ProducerPool(config) val handler = new DefaultEventHandler[Int,String](config, - partitioner = intPartitioner, - encoder = null.asInstanceOf[Encoder[String]], - keyEncoder = new IntEncoder(), - producerPool = producerPool, - topicPartitionInfos) + partitioner = intPartitioner, + encoder = null.asInstanceOf[Encoder[String]], + keyEncoder = new IntEncoder(), + producerPool = producerPool, + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) val topic1Broker1Data = ArrayBuffer[KeyedMessage[Int,Message]](new KeyedMessage[Int,Message]("topic1", 0, new Message("msg1".getBytes)), @@ -228,8 +233,9 @@ class AsyncProducerTest extends JUnit3Suite { encoder = new StringEncoder, keyEncoder = new StringEncoder, producerPool = producerPool, - topicPartitionInfos - ) + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) val serializedData = handler.serialize(produceData) val deserializedData = serializedData.map(d => new KeyedMessage[String,String](d.topic, Utils.readString(d.message.payload))) @@ -257,7 +263,9 @@ class AsyncProducerTest extends JUnit3Suite { encoder = null.asInstanceOf[Encoder[String]], keyEncoder = null.asInstanceOf[Encoder[String]], producerPool = producerPool, - topicPartitionInfos) + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) try { handler.partitionAndCollate(producerDataList) fail("Should fail with UnknownTopicOrPartitionException") @@ -288,7 +296,9 @@ class AsyncProducerTest extends JUnit3Suite { encoder = new StringEncoder, keyEncoder = new StringEncoder, producerPool = producerPool, - topicPartitionInfos) + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) try { handler.handle(producerDataList) fail("Should fail with NoBrokersForPartitionException") @@ -335,7 +345,9 @@ class AsyncProducerTest extends JUnit3Suite { encoder = null.asInstanceOf[Encoder[String]], keyEncoder = null.asInstanceOf[Encoder[String]], producerPool = producerPool, - topicPartitionInfos) + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) val producerDataList = new ArrayBuffer[KeyedMessage[String,Message]] producerDataList.append(new KeyedMessage[String,Message]("topic1", new Message("msg1".getBytes))) producerDataList.append(new KeyedMessage[String,Message]("topic2", new Message("msg2".getBytes))) @@ -373,14 +385,19 @@ class AsyncProducerTest extends JUnit3Suite { val msgs = TestUtils.getMsgStrings(10) - val handler = new DefaultEventHandler[String,String]( config, - partitioner = null.asInstanceOf[Partitioner[String]], - encoder = new StringEncoder, - keyEncoder = new StringEncoder, - producerPool = producerPool, - topicPartitionInfos) - - val producer = new Producer[String, String](config, handler) + val handler = new DefaultEventHandler[String,String](config, + partitioner = null.asInstanceOf[Partitioner[String]], + encoder = new StringEncoder, + keyEncoder = new StringEncoder, + producerPool = producerPool, + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) + + val producer = new Producer[String, String](config, + handler, + new ProducerStats(""), + new ProducerTopicStats("")) try { // send all 10 messages, should create 2 batches and 2 syncproducer calls producer.send(msgs.map(m => new KeyedMessage[String,String](topic, m)): _*) @@ -411,17 +428,18 @@ class AsyncProducerTest extends JUnit3Suite { // produce request for topic1 and partitions 0 and 1. Let the first request fail // entirely. The second request will succeed for partition 1 but fail for partition 0. // On the third try for partition 0, let it succeed. - val request1 = TestUtils.produceRequestWithAcks(List(topic1), List(0, 1), messagesToSet(msgs), 0) - val response1 = ProducerResponse(ProducerRequest.CurrentVersion, 0, + val request1 = TestUtils.produceRequestWithAcks(List(topic1), List(0, 1), messagesToSet(msgs), acks = 0, correlationId = 0) + val request2 = TestUtils.produceRequestWithAcks(List(topic1), List(0, 1), messagesToSet(msgs), acks = 0, correlationId = 1) + val response1 = ProducerResponse(0, Map((TopicAndPartition("topic1", 0), ProducerResponseStatus(ErrorMapping.NotLeaderForPartitionCode.toShort, 0L)), (TopicAndPartition("topic1", 1), ProducerResponseStatus(ErrorMapping.NoError, 0L)))) - val request2 = TestUtils.produceRequest(topic1, 0, messagesToSet(msgs)) - val response2 = ProducerResponse(ProducerRequest.CurrentVersion, 0, + val request3 = TestUtils.produceRequest(topic1, 0, messagesToSet(msgs), correlationId = 2) + val response2 = ProducerResponse(0, Map((TopicAndPartition("topic1", 0), ProducerResponseStatus(ErrorMapping.NoError, 0L)))) val mockSyncProducer = EasyMock.createMock(classOf[SyncProducer]) EasyMock.expect(mockSyncProducer.send(request1)).andThrow(new RuntimeException) // simulate SocketTimeoutException - EasyMock.expect(mockSyncProducer.send(request1)).andReturn(response1) - EasyMock.expect(mockSyncProducer.send(request2)).andReturn(response2) + EasyMock.expect(mockSyncProducer.send(request2)).andReturn(response1) + EasyMock.expect(mockSyncProducer.send(request3)).andReturn(response2) EasyMock.replay(mockSyncProducer) val producerPool = EasyMock.createMock(classOf[ProducerPool]) @@ -436,7 +454,9 @@ class AsyncProducerTest extends JUnit3Suite { encoder = new StringEncoder(), keyEncoder = new NullEncoder[Int](), producerPool = producerPool, - topicPartitionInfos) + topicPartitionInfos = topicPartitionInfos, + producerStats = new ProducerStats(""), + producerTopicStats = new ProducerTopicStats("")) val data = msgs.map(m => new KeyedMessage[Int,String](topic1, 0, m)) ++ msgs.map(m => new KeyedMessage[Int,String](topic1, 1, m)) handler.handle(data) handler.close() @@ -492,7 +512,7 @@ class AsyncProducerTest extends JUnit3Suite { } private def getTopicMetadata(topic: String, partition: Seq[Int], brokerId: Int, brokerHost: String, brokerPort: Int): TopicMetadata = { - val broker1 = new Broker(brokerId, brokerHost, brokerHost, brokerPort) + val broker1 = new Broker(brokerId, brokerHost, brokerPort) new TopicMetadata(topic, partition.map(new PartitionMetadata(_, Some(broker1), List(broker1)))) } diff --git a/core/src/test/scala/unit/kafka/producer/ProducerTest.scala b/core/src/test/scala/unit/kafka/producer/ProducerTest.scala index 0b86777..48842eb 100644 --- a/core/src/test/scala/unit/kafka/producer/ProducerTest.scala +++ b/core/src/test/scala/unit/kafka/producer/ProducerTest.scala @@ -65,8 +65,9 @@ class ProducerTest extends JUnit3Suite with ZooKeeperTestHarness with Logging{ props.put("host", "localhost") props.put("port", port1.toString) - consumer1 = new SimpleConsumer("localhost", port1, 1000000, 64*1024) - consumer2 = new SimpleConsumer("localhost", port2, 100, 64*1024) + consumer1 = new SimpleConsumer("localhost", port1, 1000000, 64*1024, "") + consumer2 = new SimpleConsumer("localhost", port2, 100, 64*1024, "") + // temporarily set request handler logger to a higher level requestHandlerLogger.setLevel(Level.FATAL) diff --git a/core/src/test/scala/unit/kafka/producer/SyncProducerTest.scala b/core/src/test/scala/unit/kafka/producer/SyncProducerTest.scala index ae34315..b289dda 100644 --- a/core/src/test/scala/unit/kafka/producer/SyncProducerTest.scala +++ b/core/src/test/scala/unit/kafka/producer/SyncProducerTest.scala @@ -81,11 +81,11 @@ class SyncProducerTest extends JUnit3Suite with KafkaServerTestHarness { props.put("connect.timeout.ms", "300") props.put("reconnect.interval", "500") props.put("max.message.size", "100") - val correlationId = SyncProducerConfig.DefaultCorrelationId + val correlationId = 0 val clientId = SyncProducerConfig.DefaultClientId val ackTimeoutMs = SyncProducerConfig.DefaultAckTimeoutMs val ack = SyncProducerConfig.DefaultRequiredAcks - val emptyRequest = new kafka.api.ProducerRequest(correlationId, clientId, ack, ackTimeoutMs, Map[TopicAndPartition, MessageSet]()) + val emptyRequest = new kafka.api.ProducerRequest(correlationId, clientId, ack, ackTimeoutMs, Map[TopicAndPartition, ByteBufferMessageSet]()) val producer = new SyncProducer(new SyncProducerConfig(props)) val response = producer.send(emptyRequest) @@ -98,9 +98,7 @@ class SyncProducerTest extends JUnit3Suite with KafkaServerTestHarness { val props = new Properties() props.put("host", "localhost") props.put("port", server.socketServer.port.toString) - props.put("buffer.size", "102400") - props.put("connect.timeout.ms", "300") - props.put("reconnect.interval", "500") + props.put("max.message.size", 50000.toString) val producer = new SyncProducer(new SyncProducerConfig(props)) CreateTopicCommand.createTopic(zkClient, "test", 1, 1) TestUtils.waitUntilLeaderIsElectedOrChanged(zkClient, "test", 0, 500) diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala index 19b90a8..fcdd26e 100644 --- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala +++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala @@ -123,7 +123,7 @@ class LeaderElectionTest extends JUnit3Suite with ZooKeeperTestHarness { // start another controller val controllerConfig = new KafkaConfig(TestUtils.createBrokerConfig(2, TestUtils.choosePort())) - val brokers = servers.map(s => new Broker(s.config.brokerId, s.config.hostName, "localhost", s.config.port)) + val brokers = servers.map(s => new Broker(s.config.brokerId, "localhost", s.config.port)) val controllerChannelManager = new ControllerChannelManager(brokers.toSet, controllerConfig) controllerChannelManager.startup() val staleControllerEpoch = 0 diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala index aa58dce..104400b 100644 --- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala +++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala @@ -54,7 +54,7 @@ class LogOffsetTest extends JUnit3Suite with ZooKeeperTestHarness { logDir = new File(logDirPath) time = new MockTime() server = TestUtils.createServer(new KafkaConfig(config), time) - simpleConsumer = new SimpleConsumer("localhost", brokerPort, 1000000, 64*1024) + simpleConsumer = new SimpleConsumer("localhost", brokerPort, 1000000, 64*1024, "") } @After @@ -93,7 +93,7 @@ class LogOffsetTest extends JUnit3Suite with ZooKeeperTestHarness { log.flush() val offsets = server.apis.fetchOffsets(logManager, TopicAndPartition(topic, part), OffsetRequest.LatestTime, 10) - assertEquals(Seq(20L, 15L, 10L, 5L, 0L), offsets) + assertEquals(Seq(20L, 16L, 12L, 8L, 4L, 0L), offsets) waitUntilTrue(() => isLeaderLocalOnBroker(topic, part, server), 1000) val topicAndPartition = TopicAndPartition(topic, part) @@ -102,7 +102,7 @@ class LogOffsetTest extends JUnit3Suite with ZooKeeperTestHarness { replicaId = 0) val consumerOffsets = simpleConsumer.getOffsetsBefore(offsetRequest).partitionErrorAndOffsets(topicAndPartition).offsets - assertEquals(Seq(20L, 15L, 10L, 5L, 0L), consumerOffsets) + assertEquals(Seq(20L, 16L, 12L, 8L, 4L, 0L), consumerOffsets) // try to fetch using latest offset val fetchResponse = simpleConsumer.fetch( @@ -156,15 +156,16 @@ class LogOffsetTest extends JUnit3Suite with ZooKeeperTestHarness { val now = time.milliseconds + 30000 // pretend it is the future to avoid race conditions with the fs + val offsets = server.apis.fetchOffsets(logManager, TopicAndPartition(topic, part), now, 10) - assertEquals(Seq(20L, 15L, 10L, 5L, 0L), offsets) + assertEquals(Seq(20L, 16L, 12L, 8L, 4L, 0L), offsets) waitUntilTrue(() => isLeaderLocalOnBroker(topic, part, server), 1000) val topicAndPartition = TopicAndPartition(topic, part) val offsetRequest = OffsetRequest(Map(topicAndPartition -> PartitionOffsetRequestInfo(now, 10)), replicaId = 0) val consumerOffsets = simpleConsumer.getOffsetsBefore(offsetRequest).partitionErrorAndOffsets(topicAndPartition).offsets - assertEquals(Seq(20L, 15L, 10L, 5L, 0L), consumerOffsets) + assertEquals(Seq(20L, 16L, 12L, 8L, 4L, 0L), consumerOffsets) } @Test diff --git a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala index ad2158c..a3afa2d 100644 --- a/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala +++ b/core/src/test/scala/unit/kafka/server/LogRecoveryTest.scala @@ -23,8 +23,6 @@ import kafka.utils.TestUtils._ import kafka.utils.IntEncoder import kafka.utils.{Utils, TestUtils} import kafka.zk.ZooKeeperTestHarness -import kafka.serializer._ -import kafka.message.Message import kafka.producer.{ProducerConfig, KeyedMessage, Producer} class LogRecoveryTest extends JUnit3Suite with ZooKeeperTestHarness { diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala index fa6a64e..7afbe54 100644 --- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala +++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala @@ -20,7 +20,7 @@ import java.io.File import kafka.consumer.SimpleConsumer import org.junit.Test import junit.framework.Assert._ -import kafka.message.{Message, ByteBufferMessageSet} +import kafka.message.ByteBufferMessageSet import org.scalatest.junit.JUnit3Suite import kafka.zk.ZooKeeperTestHarness import kafka.producer._ @@ -66,10 +66,7 @@ class ServerShutdownTest extends JUnit3Suite with ZooKeeperTestHarness { server.startup() producer = new Producer[Int, String](new ProducerConfig(producerConfig)) - val consumer = new SimpleConsumer(host, - port, - 1000000, - 64*1024) + val consumer = new SimpleConsumer(host, port, 1000000, 64*1024, "") waitUntilLeaderIsElectedOrChanged(zkClient, topic, 0, 1000) diff --git a/core/src/test/scala/unit/kafka/utils/ClientIdTest.scala b/core/src/test/scala/unit/kafka/utils/ClientIdTest.scala new file mode 100644 index 0000000..794dcdc --- /dev/null +++ b/core/src/test/scala/unit/kafka/utils/ClientIdTest.scala @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.utils + +import junit.framework.Assert._ +import collection.mutable.ArrayBuffer +import kafka.common.InvalidClientIdException +import org.junit.Test + +class ClientIdTest { + + @Test + def testInvalidClientIds() { + val invalidclientIds = new ArrayBuffer[String]() + var longName = "ATCG" + for (i <- 1 to 6) + longName += longName + invalidclientIds += longName + val badChars = Array('/', '\\', ',', '\0', ':', "\"", '\'', ';', '*', '?', '.', ' ', '\t', '\r', '\n', '=') + for (weirdChar <- badChars) { + invalidclientIds += "Is" + weirdChar + "funny" + } + + for (i <- 0 until invalidclientIds.size) { + try { + ClientId.validate(invalidclientIds(i)) + fail("Should throw InvalidClientIdException.") + } + catch { + case e: InvalidClientIdException => "This is good." + } + } + + val validClientIds = new ArrayBuffer[String]() + validClientIds += ("valid", "CLIENT", "iDs", "ar6", "VaL1d", "_0-9_", "") + for (i <- 0 until validClientIds.size) { + try { + ClientId.validate(validClientIds(i)) + } + catch { + case e: Exception => fail("Should not throw exception.") + } + } + } +} \ No newline at end of file diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index 4ef5bd4..8118d4c 100644 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -24,6 +24,7 @@ import java.nio.channels._ import java.util.Random import java.util.Properties import junit.framework.Assert._ +import kafka.api._ import kafka.server._ import kafka.producer._ import kafka.message._ @@ -332,13 +333,13 @@ object TestUtils extends Logging { } def createBrokersInZk(zkClient: ZkClient, ids: Seq[Int]): Seq[Broker] = { - val brokers = ids.map(id => new Broker(id, "localhost" + System.currentTimeMillis(), "localhost", 6667)) - brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, b.id, b.host, b.creatorId, b.port)) + val brokers = ids.map(id => new Broker(id, "localhost", 6667)) + brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, b.id, b.host, b.port)) brokers } def deleteBrokersInZk(zkClient: ZkClient, ids: Seq[Int]): Seq[Broker] = { - val brokers = ids.map(id => new Broker(id, "localhost" + System.currentTimeMillis(), "localhost", 6667)) + val brokers = ids.map(id => new Broker(id, "localhost", 6667)) brokers.foreach(b => ZkUtils.deletePath(zkClient, ZkUtils.BrokerIdsPath + "/" + b)) brokers } @@ -353,22 +354,27 @@ object TestUtils extends Logging { /** * Create a wired format request based on simple basic information */ - def produceRequest(topic: String, partition: Int, message: ByteBufferMessageSet): kafka.api.ProducerRequest = { - produceRequest(SyncProducerConfig.DefaultCorrelationId,topic,partition,message) - } - - def produceRequest(correlationId: Int, topic: String, partition: Int, message: ByteBufferMessageSet): kafka.api.ProducerRequest = { - produceRequestWithAcks(List(topic), List(partition), message, SyncProducerConfig.DefaultRequiredAcks) - } - - def produceRequestWithAcks(topics: Seq[String], partitions: Seq[Int], message: ByteBufferMessageSet, acks: Int): kafka.api.ProducerRequest = { - val correlationId = SyncProducerConfig.DefaultCorrelationId - val clientId = SyncProducerConfig.DefaultClientId - val ackTimeoutMs = SyncProducerConfig.DefaultAckTimeoutMs + def produceRequest(topic: String, + partition: Int, + message: ByteBufferMessageSet, + acks: Int = SyncProducerConfig.DefaultRequiredAcks, + timeout: Int = SyncProducerConfig.DefaultAckTimeoutMs, + correlationId: Int = 0, + clientId: String = SyncProducerConfig.DefaultClientId): ProducerRequest = { + produceRequestWithAcks(Seq(topic), Seq(partition), message, acks, timeout, correlationId, clientId) + } + + def produceRequestWithAcks(topics: Seq[String], + partitions: Seq[Int], + message: ByteBufferMessageSet, + acks: Int = SyncProducerConfig.DefaultRequiredAcks, + timeout: Int = SyncProducerConfig.DefaultAckTimeoutMs, + correlationId: Int = 0, + clientId: String = SyncProducerConfig.DefaultClientId): ProducerRequest = { val data = topics.flatMap(topic => partitions.map(partition => (TopicAndPartition(topic, partition), message)) ) - new kafka.api.ProducerRequest(correlationId, clientId, acks.toShort, ackTimeoutMs, Map(data:_*)) + new ProducerRequest(correlationId, clientId, acks.toShort, timeout, Map(data:_*)) } def makeLeaderForPartition(zkClient: ZkClient, topic: String, diff --git a/examples/src/main/java/kafka/examples/SimpleConsumerDemo.java b/examples/src/main/java/kafka/examples/SimpleConsumerDemo.java index b87d50c..c79192c 100644 --- a/examples/src/main/java/kafka/examples/SimpleConsumerDemo.java +++ b/examples/src/main/java/kafka/examples/SimpleConsumerDemo.java @@ -59,7 +59,8 @@ public class SimpleConsumerDemo { SimpleConsumer simpleConsumer = new SimpleConsumer(KafkaProperties.kafkaServerURL, KafkaProperties.kafkaServerPort, KafkaProperties.connectionTimeOut, - KafkaProperties.kafkaProducerBufferSize); + KafkaProperties.kafkaProducerBufferSize, + KafkaProperties.clientId); System.out.println("Testing single fetch"); FetchRequest req = new FetchRequestBuilder() diff --git a/perf/src/main/scala/kafka/perf/ProducerPerformance.scala b/perf/src/main/scala/kafka/perf/ProducerPerformance.scala index a270289..a9a5f07 100644 --- a/perf/src/main/scala/kafka/perf/ProducerPerformance.scala +++ b/perf/src/main/scala/kafka/perf/ProducerPerformance.scala @@ -126,11 +126,6 @@ object ProducerPerformance extends Logging { .withRequiredArg() .ofType(classOf[java.lang.Integer]) .defaultsTo(-1) - val asyncOpt = parser.accepts("async", "If set, messages are sent asynchronously.") - .withRequiredArg - .describedAs("count") - .ofType(classOf[java.lang.Integer]) - .defaultsTo(1) val csvMetricsReporterEnabledOpt = parser.accepts("csv-reporter-enabled", "If set, the CSV metrics reporter will be enabled") val metricsDirectoryOpt = parser.accepts("metrics-dir", "If csv-reporter-enable is set, and this parameter is" + "set, the csv metrics will be outputed here") @@ -223,6 +218,7 @@ object ProducerPerformance extends Logging { private val threadIdLabel = "ThreadID" private val topicLabel = "Topic" private var leftPaddedSeqId : String = "" + private def generateMessageWithSeqId(topic: String, msgId: Long, msgSize: Int): Array[Byte] = { // Each thread gets a unique range of sequential no. for its ids. // Eg. 1000 msg in 10 threads => 100 msg per thread @@ -246,12 +242,13 @@ object ProducerPerformance extends Logging { private def generateProducerData(topic: String, messageId: Long): (KeyedMessage[Long, Array[Byte]], Int) = { val msgSize = if(config.isFixSize) config.messageSize else 1 + rand.nextInt(config.messageSize) - val message = if(config.seqIdMode) { - val seqId = config.initialMessageId + (messagesPerThread * threadId) + messageId - generateMessageWithSeqId(topic, seqId, msgSize) - } else { - new Array[Byte](msgSize) - } + val message = + if(config.seqIdMode) { + val seqId = config.initialMessageId + (messagesPerThread * threadId) + messageId + generateMessageWithSeqId(topic, seqId, msgSize) + } else { + new Array[Byte](msgSize) + } (new KeyedMessage[Long, Array[Byte]](topic, messageId, message), message.length) } diff --git a/perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala b/perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala index ed1e0e8..9c9eead 100644 --- a/perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala +++ b/perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala @@ -42,7 +42,7 @@ object SimpleConsumerPerformance { println("time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec") } - val consumer = new SimpleConsumer(config.url.getHost, config.url.getPort, 30*1000, 2*config.fetchSize) + val consumer = new SimpleConsumer(config.url.getHost, config.url.getPort, 30*1000, 2*config.fetchSize, config.clientId) // reset to latest or smallest offset val topicAndPartition = TopicAndPartition(config.topic, config.partition) diff --git a/system_test/migration_tool_testsuite/migration_tool_test.py b/system_test/migration_tool_testsuite/migration_tool_test.py index d74418d..5d22f83 100644 --- a/system_test/migration_tool_testsuite/migration_tool_test.py +++ b/system_test/migration_tool_testsuite/migration_tool_test.py @@ -123,21 +123,21 @@ class MigrationToolTest(ReplicationUtils, SetupUtils): # initialize signal handler signal.signal(signal.SIGINT, self.signal_handler) - # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase - # for collecting logs from remote machines - kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) - # TestcaseEnv.testcaseConfigsList initialized by reading testcase properties file: # system_test/_testsuite/testcase_/testcase__properties.json self.testcaseEnv.testcaseConfigsList = system_test_utils.get_json_list_data( self.testcaseEnv.testcasePropJsonPathName) - # TestcaseEnv - initialize producer & consumer config / log file pathnames - kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) - # clean up data directories specified in zookeeper.properties and kafka_server_.properties kafka_system_test_utils.cleanup_data_at_remote_hosts(self.systemTestEnv, self.testcaseEnv) + # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase + # for collecting logs from remote machines + kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) + + # TestcaseEnv - initialize producer & consumer config / log file pathnames + kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) + # generate remote hosts log/config dirs if not exist kafka_system_test_utils.generate_testcase_log_dirs_in_remote_hosts(self.systemTestEnv, self.testcaseEnv) diff --git a/system_test/mirror_maker_testsuite/mirror_maker_test.py b/system_test/mirror_maker_testsuite/mirror_maker_test.py index 7445e54..48b0d25 100644 --- a/system_test/mirror_maker_testsuite/mirror_maker_test.py +++ b/system_test/mirror_maker_testsuite/mirror_maker_test.py @@ -123,22 +123,21 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils): # initialize signal handler signal.signal(signal.SIGINT, self.signal_handler) - - # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase - # for collecting logs from remote machines - kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) - # TestcaseEnv.testcaseConfigsList initialized by reading testcase properties file: # system_test/_testsuite/testcase_/testcase__properties.json self.testcaseEnv.testcaseConfigsList = system_test_utils.get_json_list_data( self.testcaseEnv.testcasePropJsonPathName) - - # TestcaseEnv - initialize producer & consumer config / log file pathnames - kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) - + # clean up data directories specified in zookeeper.properties and kafka_server_.properties kafka_system_test_utils.cleanup_data_at_remote_hosts(self.systemTestEnv, self.testcaseEnv) + # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase + # for collecting logs from remote machines + kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) + + # TestcaseEnv - initialize producer & consumer config / log file pathnames + kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) + # generate remote hosts log/config dirs if not exist kafka_system_test_utils.generate_testcase_log_dirs_in_remote_hosts(self.systemTestEnv, self.testcaseEnv) diff --git a/system_test/replication_testsuite/replica_basic_test.py b/system_test/replication_testsuite/replica_basic_test.py index 6f2e15a..3fc47d9 100644 --- a/system_test/replication_testsuite/replica_basic_test.py +++ b/system_test/replication_testsuite/replica_basic_test.py @@ -128,6 +128,11 @@ class ReplicaBasicTest(ReplicationUtils, SetupUtils): consumerMultiTopicsMode = self.testcaseEnv.testcaseArgumentsDict["consumer_multi_topics_mode"] except: pass + autoCreateTopic = "false" + try: + autoCreateTopic = self.testcaseEnv.testcaseArgumentsDict["auto_create_topic"] + except: + pass # initialize self.testcaseEnv with user-defined environment variables (product specific) @@ -139,22 +144,21 @@ class ReplicaBasicTest(ReplicationUtils, SetupUtils): # initialize signal handler signal.signal(signal.SIGINT, self.signal_handler) - # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase - # for collecting logs from remote machines - kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) - # TestcaseEnv.testcaseConfigsList initialized by reading testcase properties file: # system_test/_testsuite/testcase_/testcase__properties.json self.testcaseEnv.testcaseConfigsList = system_test_utils.get_json_list_data( self.testcaseEnv.testcasePropJsonPathName) - - # TestcaseEnv - initialize producer & consumer config / log file pathnames - kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) - # clean up data directories specified in zookeeper.properties and kafka_server_.properties kafka_system_test_utils.cleanup_data_at_remote_hosts(self.systemTestEnv, self.testcaseEnv) + # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase + # for collecting logs from remote machines + kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) + + # TestcaseEnv - initialize producer & consumer config / log file pathnames + kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) + # generate remote hosts log/config dirs if not exist kafka_system_test_utils.generate_testcase_log_dirs_in_remote_hosts(self.systemTestEnv, self.testcaseEnv) @@ -180,10 +184,11 @@ class ReplicaBasicTest(ReplicationUtils, SetupUtils): self.anonLogger.info("sleeping for 5s") time.sleep(5) - self.log_message("creating topics") - kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) - self.anonLogger.info("sleeping for 5s") - time.sleep(5) + if autoCreateTopic.lower() == "false": + self.log_message("creating topics") + kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 5s") + time.sleep(5) # ============================================= # start ConsoleConsumer if this is a Log Retention test @@ -392,7 +397,7 @@ class ReplicaBasicTest(ReplicationUtils, SetupUtils): if logRetentionTest.lower() == "false": self.log_message("starting consumer in the background") kafka_system_test_utils.start_console_consumer(self.systemTestEnv, self.testcaseEnv) - time.sleep(1) + time.sleep(10) # ============================================= # this testcase is completed - stop all entities @@ -421,11 +426,11 @@ class ReplicaBasicTest(ReplicationUtils, SetupUtils): kafka_system_test_utils.validate_simple_consumer_data_matched_across_replicas(self.systemTestEnv, self.testcaseEnv) kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) elif consumerMultiTopicsMode.lower() == "true": - kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) + #kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) kafka_system_test_utils.validate_data_matched_in_multi_topics_from_single_consumer_producer(self.systemTestEnv, self.testcaseEnv) else: kafka_system_test_utils.validate_simple_consumer_data_matched_across_replicas(self.systemTestEnv, self.testcaseEnv) - kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) + #kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) # ============================================= diff --git a/system_test/replication_testsuite/testcase_9051/cluster_config.json b/system_test/replication_testsuite/testcase_9051/cluster_config.json index 7103040..7bef92b 100644 --- a/system_test/replication_testsuite/testcase_9051/cluster_config.json +++ b/system_test/replication_testsuite/testcase_9051/cluster_config.json @@ -2,105 +2,56 @@ "cluster_config": [ { "entity_id": "0", - "hostname": "localhost", + "hostname": "esv4-app18.corp", "role": "zookeeper", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9900" }, - - { "entity_id": "1", - "hostname": "localhost", + "hostname": "esv4-app18.corp", "role": "broker", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9901" }, { "entity_id": "2", - "hostname": "localhost", + "hostname": "esv4-app19.corp", "role": "broker", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9902" }, { "entity_id": "3", - "hostname": "localhost", + "hostname": "esv4-app20.corp", "role": "broker", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9903" }, { "entity_id": "4", - "hostname": "localhost", - "role": "broker", - "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", - "jmx_port": "9904" - }, - { - "entity_id": "5", - "hostname": "localhost", - "role": "broker", - "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", - "jmx_port": "9905" - }, - { - "entity_id": "6", - "hostname": "localhost", - "role": "broker", - "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", - "jmx_port": "9906" - }, - { - "entity_id": "7", - "hostname": "localhost", - "role": "broker", - "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", - "jmx_port": "9907" - }, - { - "entity_id": "8", - "hostname": "localhost", - "role": "broker", - "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", - "jmx_port": "9908" - }, - - - { - "entity_id": "9", - "hostname": "localhost", + "hostname": "esv4-app18.corp", "role": "producer_performance", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9909" }, { - "entity_id": "10", - "hostname": "localhost", + "entity_id": "5", + "hostname": "esv4-app18.corp", "role": "console_consumer", "cluster_name": "source", - "kafka_home": "default", - "java_home": "default", + "kafka_home": "/mnt/u001/kafka_08_replication_system_test", + "java_home": "/export/apps/jdk/JDK-1_6_0_21", "jmx_port": "9910" } ] diff --git a/system_test/replication_testsuite/testcase_9051/testcase_9051_properties.json b/system_test/replication_testsuite/testcase_9051/testcase_9051_properties.json index 84e5789..9a99c39 100644 --- a/system_test/replication_testsuite/testcase_9051/testcase_9051_properties.json +++ b/system_test/replication_testsuite/testcase_9051/testcase_9051_properties.json @@ -13,7 +13,7 @@ "testcase_args": { "broker_type": "leader", "bounce_broker": "false", - "replica_factor": "8", + "replica_factor": "3", "num_partition": "2", "num_iteration": "1", "producer_multi_topics_mode": "true", @@ -59,51 +59,6 @@ }, { "entity_id": "4", - "port": "9094", - "brokerid": "4", - "log.file.size": "1048576", - "log.dir": "/tmp/kafka_server_4_logs", - "log_filename": "kafka_server_9094.log", - "config_filename": "kafka_server_9094.properties" - }, - { - "entity_id": "5", - "port": "9095", - "brokerid": "5", - "log.file.size": "1048576", - "log.dir": "/tmp/kafka_server_5_logs", - "log_filename": "kafka_server_9095.log", - "config_filename": "kafka_server_9095.properties" - }, - { - "entity_id": "6", - "port": "9096", - "brokerid": "6", - "log.file.size": "1048576", - "log.dir": "/tmp/kafka_server_6_logs", - "log_filename": "kafka_server_9096.log", - "config_filename": "kafka_server_9096.properties" - }, - { - "entity_id": "7", - "port": "9097", - "brokerid": "7", - "log.file.size": "1048576", - "log.dir": "/tmp/kafka_server_7_logs", - "log_filename": "kafka_server_9097.log", - "config_filename": "kafka_server_9097.properties" - }, - { - "entity_id": "8", - "port": "9098", - "brokerid": "8", - "log.file.size": "1048576", - "log.dir": "/tmp/kafka_server_8_logs", - "log_filename": "kafka_server_9098.log", - "config_filename": "kafka_server_9098.properties" - }, - { - "entity_id": "9", "topic": "t001,t002,t003,t004,t005,t006,t007,t008,t009,t010,t011,t012,t013,t014,t015,t016,t017,t018,t019,t020", "threads": "5", "compression-codec": "0", @@ -111,13 +66,13 @@ "message": "500", "request-num-acks": "-1", "producer-retry-backoff-ms": "3500", - "producer-num-retries": "5", + "producer-num-retries": "3", "async":"false", "log_filename": "producer_performance_9.log", "config_filename": "producer_performance_9.properties" }, { - "entity_id": "10", + "entity_id": "5", "topic": "t001,t002,t003,t004,t005,t006,t007,t008,t009,t010,t011,t012,t013,t014,t015,t016,t017,t018,t019,t020", "groupid": "mytestgroup", "consumer-timeout-ms": "60000", diff --git a/system_test/testcase_to_run.json b/system_test/testcase_to_run.json index c6cf17e..8252860 100644 --- a/system_test/testcase_to_run.json +++ b/system_test/testcase_to_run.json @@ -1,5 +1,5 @@ { "ReplicaBasicTest" : [ - "testcase_1" + "testcase_0001" ] } diff --git a/system_test/utils/kafka_system_test_utils.py b/system_test/utils/kafka_system_test_utils.py index aeb4bdf..c429833 100644 --- a/system_test/utils/kafka_system_test_utils.py +++ b/system_test/utils/kafka_system_test_utils.py @@ -131,6 +131,7 @@ def collect_logs_from_remote_hosts(systemTestEnv, testcaseEnv): hostname = clusterEntityConfigDict["hostname"] entity_id = clusterEntityConfigDict["entity_id"] role = clusterEntityConfigDict["role"] + kafkaHome = clusterEntityConfigDict["kafka_home"] logger.debug("entity_id : " + entity_id, extra=d) logger.debug("hostname : " + hostname, extra=d) @@ -139,12 +140,19 @@ def collect_logs_from_remote_hosts(systemTestEnv, testcaseEnv): configPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "config") metricsPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "metrics") logPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "default") + rmtLogPathName = logPathName + rmtMetricsPathName = metricsPathName + + if hostname != "localhost": + rmtConfigPathName = replace_kafka_home(configPathName, kafkaHome) + rmtMetricsPathName = replace_kafka_home(metricsPathName, kafkaHome) + rmtLogPathName = replace_kafka_home(logPathName, kafkaHome) # ============================== # collect entity log file # ============================== cmdList = ["scp", - hostname + ":" + logPathName + "/*", + hostname + ":" + rmtLogPathName + "/*", logPathName] cmdStr = " ".join(cmdList) logger.debug("executing command [" + cmdStr + "]", extra=d) @@ -154,7 +162,7 @@ def collect_logs_from_remote_hosts(systemTestEnv, testcaseEnv): # collect entity metrics file # ============================== cmdList = ["scp", - hostname + ":" + metricsPathName + "/*", + hostname + ":" + rmtMetricsPathName + "/*", metricsPathName] cmdStr = " ".join(cmdList) logger.debug("executing command [" + cmdStr + "]", extra=d) @@ -192,8 +200,13 @@ def collect_logs_from_remote_hosts(systemTestEnv, testcaseEnv): # collect dashboards file # ============================== dashboardsPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "dashboards") + rmtDashboardsPathName = dashboardsPathName + + if hostname != "localhost": + rmtDashboardsPathName = replace_kafka_home(dashboardsPathName, kafkaHome) + cmdList = ["scp", - hostname + ":" + dashboardsPathName + "/*", + hostname + ":" + rmtDashboardsPathName + "/*", dashboardsPathName] cmdStr = " ".join(cmdList) logger.debug("executing command [" + cmdStr + "]", extra=d) @@ -207,6 +220,7 @@ def generate_testcase_log_dirs_in_remote_hosts(systemTestEnv, testcaseEnv): hostname = clusterEntityConfigDict["hostname"] entity_id = clusterEntityConfigDict["entity_id"] role = clusterEntityConfigDict["role"] + kafkaHome = clusterEntityConfigDict["kafka_home"] logger.debug("entity_id : " + entity_id, extra=d) logger.debug("hostname : " + hostname, extra=d) @@ -216,6 +230,11 @@ def generate_testcase_log_dirs_in_remote_hosts(systemTestEnv, testcaseEnv): metricsPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "metrics") dashboardsPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entity_id, "dashboards") + if hostname != "localhost": + configPathName = replace_kafka_home(configPathName, kafkaHome) + metricsPathName = replace_kafka_home(metricsPathName, kafkaHome) + dashboardsPathName = replace_kafka_home(dashboardsPathName, kafkaHome) + cmdList = ["ssh " + hostname, "'mkdir -p", configPathName, @@ -443,9 +462,14 @@ def scp_file_to_remote_host(clusterEntityConfigDictList, testcaseEnv): for clusterEntityConfigDict in clusterEntityConfigDictList: hostname = clusterEntityConfigDict["hostname"] - testcasePathName = testcaseEnv.testCaseBaseDir + kafkaHome = clusterEntityConfigDict["kafka_home"] + localTestcasePathName = testcaseEnv.testCaseBaseDir + remoteTestcasePathName = localTestcasePathName + + if hostname != "localhost": + remoteTestcasePathName = replace_kafka_home(localTestcasePathName, kafkaHome) - cmdStr = "scp " + testcasePathName + "/config/* " + hostname + ":" + testcasePathName + "/config" + cmdStr = "scp " + localTestcasePathName + "/config/* " + hostname + ":" + remoteTestcasePathName + "/config" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) @@ -586,10 +610,16 @@ def get_leader_elected_log_line(systemTestEnv, testcaseEnv, leaderAttributesDict hostname = system_test_utils.get_data_by_lookup_keyval( \ clusterEntityConfigDictList, "entity_id", brokerEntityId, "hostname") + kafkaHome = system_test_utils.get_data_by_lookup_keyval( \ + clusterEntityConfigDictList, "entity_id", brokerEntityId, "kafka_home") logFile = system_test_utils.get_data_by_lookup_keyval( \ testcaseEnv.testcaseConfigsList, "entity_id", brokerEntityId, "log_filename") logPathName = get_testcase_config_log_dir_pathname(testcaseEnv, "broker", brokerEntityId, "default") + + if hostname != "localhost": + logPathName = replace_kafka_home(logPathName, kafkaHome) + cmdStrList = ["ssh " + hostname, "\"grep -i -h '" + leaderAttributesDict["LEADER_ELECTION_COMPLETED_MSG"] + "' ", logPathName + "/" + logFile + " | ", @@ -659,6 +689,10 @@ def start_entity_in_background(systemTestEnv, testcaseEnv, entityId): configPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entityId, "config") logPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entityId, "default") + if hostname != "localhost": + configPathName = replace_kafka_home(configPathName, kafkaHome) + logPathName = replace_kafka_home(logPathName, kafkaHome) + if role == "zookeeper": cmdList = ["ssh " + hostname, "'JAVA_HOME=" + javaHome, @@ -728,10 +762,15 @@ def start_console_consumer(systemTestEnv, testcaseEnv): jmxPort = system_test_utils.get_data_by_lookup_keyval(clusterList, "entity_id", entityId, "jmx_port") kafkaRunClassBin = kafkaHome + "/bin/kafka-run-class.sh" - logger.info("starting console consumer", extra=d) - consumerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "default") + consumerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "default") + metricsDir = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "metrics"), + + if host != "localhost": + consumerLogPath = replace_kafka_home(consumerLogPath, kafkaHome) + #metricsDir = replace_kafka_home(metricsDir, kafkaHome) + consumerLogPathName = consumerLogPath + "/console_consumer.log" testcaseEnv.userDefinedEnvVarDict["consumerLogPathName"] = consumerLogPathName @@ -741,7 +780,6 @@ def start_console_consumer(systemTestEnv, testcaseEnv): topic = system_test_utils.get_data_by_lookup_keyval(testcaseList, "entity_id", entityId, "topic") timeoutMs = system_test_utils.get_data_by_lookup_keyval(testcaseList, "entity_id", entityId, "consumer-timeout-ms") - formatterOption = "" try: formatterOption = system_test_utils.get_data_by_lookup_keyval(testcaseList, "entity_id", entityId, "formatter") @@ -768,7 +806,7 @@ def start_console_consumer(systemTestEnv, testcaseEnv): "--topic " + topic, "--consumer-timeout-ms " + timeoutMs, "--csv-reporter-enabled", - "--metrics-dir " + get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "metrics"), + #"--metrics-dir " + metricsDir, formatterOption, "--from-beginning ", " >> " + consumerLogPathName, @@ -860,7 +898,13 @@ def start_producer_in_thread(testcaseEnv, entityConfigList, producerConfig, kafk logger.info("starting producer preformance", extra=d) - producerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "producer_performance", entityId, "default") + producerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "producer_performance", entityId, "default") + metricsDir = get_testcase_config_log_dir_pathname(testcaseEnv, "producer_performance", entityId, "metrics") + + if host != "localhost": + producerLogPath = replace_kafka_home(producerLogPath, kafkaHome) + metricsDir = replace_kafka_home(metricsDir, kafkaHome) + producerLogPathName = producerLogPath + "/producer_performance.log" testcaseEnv.userDefinedEnvVarDict["producerLogPathName"] = producerLogPathName @@ -898,7 +942,7 @@ def start_producer_in_thread(testcaseEnv, entityConfigList, producerConfig, kafk "--producer-retry-backoff-ms " + retryBackoffMs, "--producer-num-retries " + numOfRetries, "--csv-reporter-enabled", - "--metrics-dir " + get_testcase_config_log_dir_pathname(testcaseEnv, "producer_performance", entityId, "metrics"), + "--metrics-dir " + metricsDir, boolArgumentsStr, " >> " + producerLogPathName, " & echo pid:$! > " + producerLogPath + "/entity_" + entityId + "_pid'"] @@ -1016,6 +1060,11 @@ def create_topic(systemTestEnv, testcaseEnv): else: raise Exception("Empty zkConnectStr found") + testcaseBaseDir = testcaseEnv.testCaseBaseDir + + if zkHost != "localhost": + testcaseBaseDir = replace_kafka_home(testcaseBaseDir, kafkaHome) + for topic in topicsList: logger.info("creating topic: [" + topic + "] at: [" + zkConnectStr + "]", extra=d) cmdList = ["ssh " + zkHost, @@ -1025,7 +1074,7 @@ def create_topic(systemTestEnv, testcaseEnv): " --zookeeper " + zkConnectStr, " --replica " + testcaseEnv.testcaseArgumentsDict["replica_factor"], " --partition " + testcaseEnv.testcaseArgumentsDict["num_partition"] + " >> ", - testcaseEnv.testCaseBaseDir + "/logs/create_source_cluster_topic.log'"] + testcaseBaseDir + "/logs/create_source_cluster_topic.log'"] cmdStr = " ".join(cmdList) logger.debug("executing command: [" + cmdStr + "]", extra=d) @@ -1159,6 +1208,23 @@ def cleanup_data_at_remote_hosts(systemTestEnv, testcaseEnv): clusterEntityConfigDictList = systemTestEnv.clusterEntityConfigDictList testcaseConfigsList = testcaseEnv.testcaseConfigsList + testCaseBaseDir = testcaseEnv.testCaseBaseDir + + # clean up the following directories in localhost + # system_test//testcase_xxxx/config + # system_test//testcase_xxxx/dashboards + # system_test//testcase_xxxx/logs + logger.info("cleaning up test case dir: [" + testCaseBaseDir + "]", extra=d) + + if "system_test" not in testCaseBaseDir: + logger.warn("possible destructive command [" + cmdStr + "]", extra=d) + logger.warn("check config file: system_test/cluster_config.properties", extra=d) + logger.warn("aborting test...", extra=d) + sys.exit(1) + else: + system_test_utils.sys_call("rm -rf " + testCaseBaseDir + "/config/*") + system_test_utils.sys_call("rm -rf " + testCaseBaseDir + "/dashboards/*") + system_test_utils.sys_call("rm -rf " + testCaseBaseDir + "/logs/*") for clusterEntityConfigDict in systemTestEnv.clusterEntityConfigDictList: @@ -1166,10 +1232,14 @@ def cleanup_data_at_remote_hosts(systemTestEnv, testcaseEnv): entityId = clusterEntityConfigDict["entity_id"] role = clusterEntityConfigDict["role"] kafkaHome = clusterEntityConfigDict["kafka_home"] - testCaseBaseDir = testcaseEnv.testCaseBaseDir cmdStr = "" dataDir = "" + if hostname == "localhost": + remoteTestCaseBaseDir = testCaseBaseDir + else: + remoteTestCaseBaseDir = replace_kafka_home(testCaseBaseDir, kafkaHome) + logger.info("cleaning up data dir on host: [" + hostname + "]", extra=d) if role == 'zookeeper': @@ -1199,26 +1269,31 @@ def cleanup_data_at_remote_hosts(systemTestEnv, testcaseEnv): # ============================ if system_test_utils.remote_host_file_exists(hostname, kafkaHome + "/bin/kafka-run-class.sh"): # so kafkaHome is a real kafka installation - cmdStr = "ssh " + hostname + " \"find " + testCaseBaseDir + " -name '*.log' | xargs rm 2> /dev/null\"" + cmdStr = "ssh " + hostname + " \"find " + remoteTestCaseBaseDir + " -name '*.log' | xargs rm 2> /dev/null\"" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) - cmdStr = "ssh " + hostname + " \"find " + testCaseBaseDir + " -name '*_pid' | xargs rm 2> /dev/null\"" + cmdStr = "ssh " + hostname + " \"find " + remoteTestCaseBaseDir + " -name '*_pid' | xargs rm 2> /dev/null\"" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) - cmdStr = "ssh " + hostname + " \"find " + testCaseBaseDir + " -name '*.csv' | xargs rm 2> /dev/null\"" + cmdStr = "ssh " + hostname + " \"find " + remoteTestCaseBaseDir + " -name '*.csv' | xargs rm 2> /dev/null\"" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) - cmdStr = "ssh " + hostname + " \"find " + testCaseBaseDir + " -name '*.svg' | xargs rm 2> /dev/null\"" + cmdStr = "ssh " + hostname + " \"find " + remoteTestCaseBaseDir + " -name '*.svg' | xargs rm 2> /dev/null\"" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) - cmdStr = "ssh " + hostname + " \"find " + testCaseBaseDir + " -name '*.html' | xargs rm 2> /dev/null\"" + cmdStr = "ssh " + hostname + " \"find " + remoteTestCaseBaseDir + " -name '*.html' | xargs rm 2> /dev/null\"" logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) +def replace_kafka_home(systemTestSubDirPath, kafkaHome): + matchObj = re.match(".*(\/system_test\/.*)$", systemTestSubDirPath) + relativeSubDirPath = matchObj.group(1) + return kafkaHome + relativeSubDirPath + def get_entity_log_directory(testCaseBaseDir, entity_id, role): return testCaseBaseDir + "/logs/" + role + "-" + entity_id @@ -1602,6 +1677,9 @@ def start_simple_consumer(systemTestEnv, testcaseEnv, minStartingOffsetDict=None kafkaRunClassBin = kafkaHome + "/bin/kafka-run-class.sh" consumerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "default") + if host != "localhost": + consumerLogPath = replace_kafka_home(consumerLogPath, kafkaHome) + # testcase configurations: testcaseList = testcaseEnv.testcaseConfigsList topic = system_test_utils.get_data_by_lookup_keyval(testcaseList, "entity_id", entityId, "topic")