From 46b3d2d392d288fe9c9693462706bea614e2b626 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 9 Apr 2015 14:29:08 -0700 Subject: [PATCH 01/24] replaced Recieve --- .../kafka/common/network/NetworkReceive.java | 34 ++++++++ .../scala/kafka/admin/ConsumerGroupCommand.scala | 2 +- core/src/main/scala/kafka/client/ClientUtils.scala | 2 +- .../main/scala/kafka/consumer/SimpleConsumer.scala | 19 ++--- .../consumer/ZookeeperConsumerConnector.scala | 4 +- .../controller/ControllerChannelManager.scala | 11 +-- .../main/scala/kafka/network/BlockingChannel.scala | 6 +- .../kafka/network/BoundedByteBufferReceive.scala | 90 ---------------------- core/src/main/scala/kafka/network/Handler.scala | 6 +- .../main/scala/kafka/network/SocketServer.scala | 8 +- .../main/scala/kafka/network/Transmission.scala | 21 +---- .../main/scala/kafka/producer/SyncProducer.scala | 13 ++-- core/src/main/scala/kafka/server/KafkaServer.scala | 8 +- .../scala/kafka/tools/ConsumerOffsetChecker.scala | 2 +- .../test/scala/other/kafka/TestOffsetManager.scala | 6 +- 15 files changed, 84 insertions(+), 148 deletions(-) delete mode 100755 core/src/main/scala/kafka/network/BoundedByteBufferReceive.scala diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index fc0d168..5207807 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -15,6 +15,7 @@ package org.apache.kafka.common.network; import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; import java.nio.channels.ScatteringByteChannel; /** @@ -22,20 +23,38 @@ import java.nio.channels.ScatteringByteChannel; */ public class NetworkReceive implements Receive { + private final static int UNKNOWN_SOURCE = -1; + private final static int UNLIMITED = -1; + private final int source; private final ByteBuffer size; + private final long maxSize; private ByteBuffer buffer; + public NetworkReceive(int source, ByteBuffer buffer) { this.source = source; this.buffer = buffer; this.size = null; + this.maxSize = UNLIMITED; } public NetworkReceive(int source) { this.source = source; this.size = ByteBuffer.allocate(4); this.buffer = null; + this.maxSize = UNLIMITED; + } + + public NetworkReceive(long maxSize) { + this.source = UNKNOWN_SOURCE; + this.size = ByteBuffer.allocate(4); + this.buffer = null; + this.maxSize = maxSize; + } + + public NetworkReceive() { + this(UNKNOWN_SOURCE); } @Override @@ -55,6 +74,12 @@ public class NetworkReceive implements Receive { @Override public long readFrom(ScatteringByteChannel channel) throws IOException { + return readFromReadableChannel(channel); + } + + // Need a method to read from ReadableByteChannel because BlockingChannel requires read with timeout + // See: http://stackoverflow.com/questions/2866557/timeout-for-socketchannel-doesnt-work + public long readFromReadableChannel(ReadableByteChannel channel) throws IOException { int read = 0; if (size.hasRemaining()) { int bytesRead = channel.read(size); @@ -83,4 +108,13 @@ public class NetworkReceive implements Receive { return this.buffer; } + // Used only by BlockingChannel, so we may be able to get rid of this when/if we get rid of BlockingChannel + public long readCompletely(ReadableByteChannel channel) throws IOException { + int totalRead = 0; + while (!complete()) { + totalRead += readFromReadableChannel(channel); + } + return totalRead; + } + } diff --git a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala index 1c3b380..05e7db1 100755 --- a/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala +++ b/core/src/main/scala/kafka/admin/ConsumerGroupCommand.scala @@ -174,7 +174,7 @@ object ConsumerGroupCommand { val offsetMap = mutable.Map[TopicAndPartition, Long]() val channel = ClientUtils.channelToOffsetManager(group, zkClient, channelSocketTimeoutMs, channelRetryBackoffMs) channel.send(OffsetFetchRequest(group, topicPartitions)) - val offsetFetchResponse = OffsetFetchResponse.readFrom(channel.receive().buffer) + val offsetFetchResponse = OffsetFetchResponse.readFrom(channel.receive().payload()) offsetFetchResponse.requestInfo.foreach { case (topicAndPartition, offsetAndMetadata) => if (offsetAndMetadata == OffsetMetadataAndError.NoOffset) { diff --git a/core/src/main/scala/kafka/client/ClientUtils.scala b/core/src/main/scala/kafka/client/ClientUtils.scala index b66424b..18529b0 100755 --- a/core/src/main/scala/kafka/client/ClientUtils.scala +++ b/core/src/main/scala/kafka/client/ClientUtils.scala @@ -153,7 +153,7 @@ object ClientUtils extends Logging{ debug("Querying %s:%d to locate offset manager for %s.".format(queryChannel.host, queryChannel.port, group)) queryChannel.send(ConsumerMetadataRequest(group)) val response = queryChannel.receive() - val consumerMetadataResponse = ConsumerMetadataResponse.readFrom(response.buffer) + val consumerMetadataResponse = ConsumerMetadataResponse.readFrom(response.payload()) debug("Consumer metadata response: " + consumerMetadataResponse.toString) if (consumerMetadataResponse.errorCode == ErrorMapping.NoError) coordinatorOpt = consumerMetadataResponse.coordinatorOpt diff --git a/core/src/main/scala/kafka/consumer/SimpleConsumer.scala b/core/src/main/scala/kafka/consumer/SimpleConsumer.scala index cbef84a..2fce6bd 100644 --- a/core/src/main/scala/kafka/consumer/SimpleConsumer.scala +++ b/core/src/main/scala/kafka/consumer/SimpleConsumer.scala @@ -21,6 +21,7 @@ import kafka.api._ import kafka.network._ import kafka.utils._ import kafka.common.{ErrorMapping, TopicAndPartition} +import org.apache.kafka.common.network.{NetworkReceive, Receive} import org.apache.kafka.common.utils.Utils._ /** @@ -62,9 +63,9 @@ class SimpleConsumer(val host: String, } } - private def sendRequest(request: RequestOrResponse): Receive = { + private def sendRequest(request: RequestOrResponse): NetworkReceive = { lock synchronized { - var response: Receive = null + var response: NetworkReceive = null try { getOrMakeConnection() blockingChannel.send(request) @@ -89,12 +90,12 @@ class SimpleConsumer(val host: String, def send(request: TopicMetadataRequest): TopicMetadataResponse = { val response = sendRequest(request) - TopicMetadataResponse.readFrom(response.buffer) + TopicMetadataResponse.readFrom(response.payload()) } def send(request: ConsumerMetadataRequest): ConsumerMetadataResponse = { val response = sendRequest(request) - ConsumerMetadataResponse.readFrom(response.buffer) + ConsumerMetadataResponse.readFrom(response.payload()) } /** @@ -104,7 +105,7 @@ class SimpleConsumer(val host: String, * @return a set of fetched messages */ def fetch(request: FetchRequest): FetchResponse = { - var response: Receive = null + var response: NetworkReceive = null val specificTimer = fetchRequestAndResponseStats.getFetchRequestAndResponseStats(host, port).requestTimer val aggregateTimer = fetchRequestAndResponseStats.getFetchRequestAndResponseAllBrokersStats.requestTimer aggregateTimer.time { @@ -112,7 +113,7 @@ class SimpleConsumer(val host: String, response = sendRequest(request) } } - val fetchResponse = FetchResponse.readFrom(response.buffer) + val fetchResponse = FetchResponse.readFrom(response.payload()) val fetchedSize = fetchResponse.sizeInBytes fetchRequestAndResponseStats.getFetchRequestAndResponseStats(host, port).requestSizeHist.update(fetchedSize) fetchRequestAndResponseStats.getFetchRequestAndResponseAllBrokersStats.requestSizeHist.update(fetchedSize) @@ -124,7 +125,7 @@ class SimpleConsumer(val host: String, * @param request a [[kafka.api.OffsetRequest]] object. * @return a [[kafka.api.OffsetResponse]] object. */ - def getOffsetsBefore(request: OffsetRequest) = OffsetResponse.readFrom(sendRequest(request).buffer) + def getOffsetsBefore(request: OffsetRequest) = OffsetResponse.readFrom(sendRequest(request).payload()) /** * Commit offsets for a topic @@ -135,7 +136,7 @@ class SimpleConsumer(val host: String, def commitOffsets(request: OffsetCommitRequest) = { // TODO: With KAFKA-1012, we have to first issue a ConsumerMetadataRequest and connect to the coordinator before // we can commit offsets. - OffsetCommitResponse.readFrom(sendRequest(request).buffer) + OffsetCommitResponse.readFrom(sendRequest(request).payload()) } /** @@ -144,7 +145,7 @@ class SimpleConsumer(val host: String, * @param request a [[kafka.api.OffsetFetchRequest]] object. * @return a [[kafka.api.OffsetFetchResponse]] object. */ - def fetchOffsets(request: OffsetFetchRequest) = OffsetFetchResponse.readFrom(sendRequest(request).buffer) + def fetchOffsets(request: OffsetFetchRequest) = OffsetFetchResponse.readFrom(sendRequest(request).payload()) private def getOrMakeConnection() { if(!isClosed && !blockingChannel.isConnected) { diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e250b94..ae5181a 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -334,7 +334,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, try { kafkaCommitMeter.mark(offsetsToCommit.size) offsetsChannel.send(offsetCommitRequest) - val offsetCommitResponse = OffsetCommitResponse.readFrom(offsetsChannel.receive().buffer) + val offsetCommitResponse = OffsetCommitResponse.readFrom(offsetsChannel.receive().payload()) trace("Offset commit response: %s.".format(offsetCommitResponse)) val (commitFailed, retryableIfFailed, shouldRefreshCoordinator, errorCount) = { @@ -421,7 +421,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, ensureOffsetManagerConnected() try { offsetsChannel.send(offsetFetchRequest) - val offsetFetchResponse = OffsetFetchResponse.readFrom(offsetsChannel.receive().buffer) + val offsetFetchResponse = OffsetFetchResponse.readFrom(offsetsChannel.receive().payload()) trace("Offset fetch response: %s.".format(offsetFetchResponse)) val (leaderChanged, loadInProgress) = diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 97acdb2..da71f07 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -16,8 +16,9 @@ */ package kafka.controller -import kafka.network.{Receive, BlockingChannel} +import kafka.network.BlockingChannel import kafka.utils.{CoreUtils, Logging, ShutdownableThread} +import org.apache.kafka.common.network.NetworkReceive import collection.mutable.HashMap import kafka.cluster.Broker import java.util.concurrent.{LinkedBlockingQueue, BlockingQueue} @@ -120,7 +121,7 @@ class RequestSendThread(val controllerId: Int, val queueItem = queue.take() val request = queueItem._1 val callback = queueItem._2 - var receive: Receive = null + var receive: NetworkReceive = null try { lock synchronized { var isSendSuccessful = false @@ -147,11 +148,11 @@ class RequestSendThread(val controllerId: Int, var response: RequestOrResponse = null request.requestId.get match { case RequestKeys.LeaderAndIsrKey => - response = LeaderAndIsrResponse.readFrom(receive.buffer) + response = LeaderAndIsrResponse.readFrom(receive.payload()) case RequestKeys.StopReplicaKey => - response = StopReplicaResponse.readFrom(receive.buffer) + response = StopReplicaResponse.readFrom(receive.payload()) case RequestKeys.UpdateMetadataKey => - response = UpdateMetadataResponse.readFrom(receive.buffer) + response = UpdateMetadataResponse.readFrom(receive.payload()) } stateChangeLogger.trace("Controller %d epoch %d received response %s for a request sent to broker %s" .format(controllerId, controllerContext.epoch, response.toString, toBroker.toString)) diff --git a/core/src/main/scala/kafka/network/BlockingChannel.scala b/core/src/main/scala/kafka/network/BlockingChannel.scala index 6e2a38e..7c0d8fc 100644 --- a/core/src/main/scala/kafka/network/BlockingChannel.scala +++ b/core/src/main/scala/kafka/network/BlockingChannel.scala @@ -21,6 +21,7 @@ import java.net.InetSocketAddress import java.nio.channels._ import kafka.utils.{nonthreadsafe, Logging} import kafka.api.RequestOrResponse +import org.apache.kafka.common.network.NetworkReceive object BlockingChannel{ @@ -103,12 +104,13 @@ class BlockingChannel( val host: String, send.writeCompletely(writeChannel) } - def receive(): Receive = { + def receive(): NetworkReceive = { if(!connected) throw new ClosedChannelException() - val response = new BoundedByteBufferReceive() + val response = new NetworkReceive() response.readCompletely(readChannel) + response.payload().rewind() response } diff --git a/core/src/main/scala/kafka/network/BoundedByteBufferReceive.scala b/core/src/main/scala/kafka/network/BoundedByteBufferReceive.scala deleted file mode 100755 index c0d7726..0000000 --- a/core/src/main/scala/kafka/network/BoundedByteBufferReceive.scala +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.network - -import java.nio._ -import java.nio.channels._ -import kafka.utils._ - -/** - * Represents a communication between the client and server - * - */ -@nonthreadsafe -private[kafka] class BoundedByteBufferReceive(val maxSize: Int) extends Receive with Logging { - - private val sizeBuffer = ByteBuffer.allocate(4) - private var contentBuffer: ByteBuffer = null - - def this() = this(Int.MaxValue) - - var complete: Boolean = false - - /** - * Get the content buffer for this transmission - */ - def buffer: ByteBuffer = { - expectComplete() - contentBuffer - } - - /** - * Read the bytes in this response from the given channel - */ - def readFrom(channel: ReadableByteChannel): Int = { - expectIncomplete() - var read = 0 - // have we read the request size yet? - if(sizeBuffer.remaining > 0) - read += CoreUtils.read(channel, sizeBuffer) - // have we allocated the request buffer yet? - if(contentBuffer == null && !sizeBuffer.hasRemaining) { - sizeBuffer.rewind() - val size = sizeBuffer.getInt() - if(size <= 0) - throw new InvalidRequestException("%d is not a valid request size.".format(size)) - if(size > maxSize) - throw new InvalidRequestException("Request of length %d is not valid, it is larger than the maximum size of %d bytes.".format(size, maxSize)) - contentBuffer = byteBufferAllocate(size) - } - // if we have a buffer read some stuff into it - if(contentBuffer != null) { - read = CoreUtils.read(channel, contentBuffer) - // did we get everything? - if(!contentBuffer.hasRemaining) { - contentBuffer.rewind() - complete = true - } - } - read - } - - private def byteBufferAllocate(size: Int): ByteBuffer = { - var buffer: ByteBuffer = null - try { - buffer = ByteBuffer.allocate(size) - } catch { - case e: OutOfMemoryError => - error("OOME with size " + size, e) - throw e - case e2: Throwable => - throw e2 - } - buffer - } -} diff --git a/core/src/main/scala/kafka/network/Handler.scala b/core/src/main/scala/kafka/network/Handler.scala index a030033..cd1d143 100644 --- a/core/src/main/scala/kafka/network/Handler.scala +++ b/core/src/main/scala/kafka/network/Handler.scala @@ -17,17 +17,19 @@ package kafka.network +import org.apache.kafka.common.network.NetworkReceive + private[kafka] object Handler { /** * A request handler is a function that turns an incoming * transmission into an outgoing transmission */ - type Handler = Receive => Option[Send] + type Handler = NetworkReceive => Option[Send] /** * A handler mapping finds the right Handler function for a given request */ - type HandlerMapping = (Short, Receive) => Handler + type HandlerMapping = (Short, NetworkReceive) => Handler } diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index c5fec00..3998eb4 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -25,6 +25,7 @@ import java.io._ import java.nio.channels._ import kafka.cluster.EndPoint +import org.apache.kafka.common.network.NetworkReceive import org.apache.kafka.common.protocol.SecurityProtocol import scala.collection._ @@ -474,9 +475,9 @@ private[kafka] class Processor(val id: Int, def read(key: SelectionKey) { lruConnections.put(key, currentTimeNanos) val socketChannel = channelFor(key) - var receive = key.attachment.asInstanceOf[Receive] + var receive = key.attachment.asInstanceOf[NetworkReceive] if(key.attachment == null) { - receive = new BoundedByteBufferReceive(maxRequestSize) + receive = new NetworkReceive(maxRequestSize.toLong) key.attach(receive) } val read = receive.readFrom(socketChannel) @@ -487,7 +488,8 @@ private[kafka] class Processor(val id: Int, } else if(receive.complete) { val port = socketChannel.socket().getLocalPort val protocol = portToProtocol.get(port) - val req = RequestChannel.Request(processor = id, requestKey = key, buffer = receive.buffer, startTimeMs = time.milliseconds, remoteAddress = address, securityProtocol = protocol) + receive.payload().rewind() + val req = RequestChannel.Request(processor = id, requestKey = key, buffer = receive.payload(), startTimeMs = time.milliseconds, remoteAddress = address, securityProtocol = protocol) requestChannel.sendRequest(req) key.attach(null) // explicitly reset interest ops to not READ, no need to wake up the selector just yet diff --git a/core/src/main/scala/kafka/network/Transmission.scala b/core/src/main/scala/kafka/network/Transmission.scala index 2827103..296a78b 100644 --- a/core/src/main/scala/kafka/network/Transmission.scala +++ b/core/src/main/scala/kafka/network/Transmission.scala @@ -41,26 +41,7 @@ private[network] trait Transmission extends Logging { } -/** - * A transmission that is being received from a channel - */ -trait Receive extends Transmission { - - def buffer: ByteBuffer - - def readFrom(channel: ReadableByteChannel): Int - - def readCompletely(channel: ReadableByteChannel): Int = { - var totalRead = 0 - while(!complete) { - val read = readFrom(channel) - trace(read + " bytes read.") - totalRead += read - } - totalRead - } - -} + /** * A transmission that is being sent out to the channel diff --git a/core/src/main/scala/kafka/producer/SyncProducer.scala b/core/src/main/scala/kafka/producer/SyncProducer.scala index 0f09951..a19058e 100644 --- a/core/src/main/scala/kafka/producer/SyncProducer.scala +++ b/core/src/main/scala/kafka/producer/SyncProducer.scala @@ -18,10 +18,11 @@ package kafka.producer import kafka.api._ -import kafka.network.{BlockingChannel, BoundedByteBufferSend, Receive} +import kafka.network.{BlockingChannel, BoundedByteBufferSend} import kafka.utils._ import java.util.Random +import org.apache.kafka.common.network.NetworkReceive import org.apache.kafka.common.utils.Utils._ object SyncProducer { @@ -63,12 +64,12 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { /** * Common functionality for the public send methods */ - private def doSend(request: RequestOrResponse, readResponse: Boolean = true): Receive = { + private def doSend(request: RequestOrResponse, readResponse: Boolean = true): NetworkReceive = { lock synchronized { verifyRequest(request) getOrMakeConnection() - var response: Receive = null + var response: NetworkReceive = null try { blockingChannel.send(request) if(readResponse) @@ -95,7 +96,7 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { producerRequestStats.getProducerRequestStats(config.host, config.port).requestSizeHist.update(requestSize) producerRequestStats.getProducerRequestAllBrokersStats.requestSizeHist.update(requestSize) - var response: Receive = null + var response: NetworkReceive = null val specificTimer = producerRequestStats.getProducerRequestStats(config.host, config.port).requestTimer val aggregateTimer = producerRequestStats.getProducerRequestAllBrokersStats.requestTimer aggregateTimer.time { @@ -104,14 +105,14 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { } } if(producerRequest.requiredAcks != 0) - ProducerResponse.readFrom(response.buffer) + ProducerResponse.readFrom(response.payload()) else null } def send(request: TopicMetadataRequest): TopicMetadataResponse = { val response = doSend(request) - TopicMetadataResponse.readFrom(response.buffer) + TopicMetadataResponse.readFrom(response.payload()) } def close() = { diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index c63f4ba..a57e0b8 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -26,13 +26,15 @@ import java.util.concurrent._ import atomic.{AtomicInteger, AtomicBoolean} import java.io.File +import org.apache.kafka.common.network.NetworkReceive + import collection.mutable import org.I0Itec.zkclient.ZkClient import kafka.controller.{ControllerStats, KafkaController} import kafka.cluster.{EndPoint, Broker} import kafka.api.{ControlledShutdownResponse, ControlledShutdownRequest} import kafka.common.{ErrorMapping, InconsistentBrokerIdException, GenerateBrokerIdException} -import kafka.network.{Receive, BlockingChannel, SocketServer} +import kafka.network.{BlockingChannel, SocketServer} import kafka.metrics.KafkaMetricsGroup import com.yammer.metrics.core.Gauge import kafka.coordinator.ConsumerCoordinator @@ -262,14 +264,14 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg // 2. issue a controlled shutdown to the controller if (channel != null) { - var response: Receive = null + var response: NetworkReceive = null try { // send the controlled shutdown request val request = new ControlledShutdownRequest(correlationId.getAndIncrement, config.brokerId) channel.send(request) response = channel.receive() - val shutdownResponse = ControlledShutdownResponse.readFrom(response.buffer) + val shutdownResponse = ControlledShutdownResponse.readFrom(response.payload()) if (shutdownResponse.errorCode == ErrorMapping.NoError && shutdownResponse.partitionsRemaining != null && shutdownResponse.partitionsRemaining.size == 0) { shutdownSucceeded = true diff --git a/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala b/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala index d2bac85..0740b3a 100644 --- a/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala +++ b/core/src/main/scala/kafka/tools/ConsumerOffsetChecker.scala @@ -162,7 +162,7 @@ object ConsumerOffsetChecker extends Logging { debug("Sending offset fetch request to coordinator %s:%d.".format(channel.host, channel.port)) channel.send(OffsetFetchRequest(group, topicPartitions)) - val offsetFetchResponse = OffsetFetchResponse.readFrom(channel.receive().buffer) + val offsetFetchResponse = OffsetFetchResponse.readFrom(channel.receive().payload()) debug("Received offset fetch response %s.".format(offsetFetchResponse)) offsetFetchResponse.requestInfo.foreach { case (topicAndPartition, offsetAndMetadata) => diff --git a/core/src/test/scala/other/kafka/TestOffsetManager.scala b/core/src/test/scala/other/kafka/TestOffsetManager.scala index 9881bd3..b708cd6 100644 --- a/core/src/test/scala/other/kafka/TestOffsetManager.scala +++ b/core/src/test/scala/other/kafka/TestOffsetManager.scala @@ -72,7 +72,7 @@ object TestOffsetManager { offsetsChannel.send(commitRequest) numCommits.getAndIncrement commitTimer.time { - val response = OffsetCommitResponse.readFrom(offsetsChannel.receive().buffer) + val response = OffsetCommitResponse.readFrom(offsetsChannel.receive().payload()) if (response.commitStatus.exists(_._2 != ErrorMapping.NoError)) numErrors.getAndIncrement } offset += 1 @@ -119,7 +119,7 @@ object TestOffsetManager { val group = "group-" + id try { metadataChannel.send(ConsumerMetadataRequest(group)) - val coordinatorId = ConsumerMetadataResponse.readFrom(metadataChannel.receive().buffer).coordinatorOpt.map(_.id).getOrElse(-1) + val coordinatorId = ConsumerMetadataResponse.readFrom(metadataChannel.receive().payload()).coordinatorOpt.map(_.id).getOrElse(-1) val channel = if (channels.contains(coordinatorId)) channels(coordinatorId) @@ -135,7 +135,7 @@ object TestOffsetManager { channel.send(fetchRequest) fetchTimer.time { - val response = OffsetFetchResponse.readFrom(channel.receive().buffer) + val response = OffsetFetchResponse.readFrom(channel.receive().payload()) if (response.requestInfo.exists(_._2.error != ErrorMapping.NoError)) { numErrors.getAndIncrement } -- 2.3.2 (Apple Git-55) From d4e2c082423f2e8b208659a5417d5b7ff08b1c1d Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 9 Apr 2015 18:11:24 -0700 Subject: [PATCH 02/24] first pass on replacing Send --- .../kafka/common/network/ByteBufferSend.java | 2 +- .../apache/kafka/common/network/NetworkSend.java | 6 ++ .../java/org/apache/kafka/common/network/Send.java | 2 +- .../apache/kafka/common/requests/ResponseSend.java | 43 +++++++++++++ .../scala/kafka/api/ConsumerMetadataRequest.scala | 7 ++- .../kafka/api/ControlledShutdownRequest.scala | 9 ++- core/src/main/scala/kafka/api/FetchResponse.scala | 49 ++++++++++----- .../main/scala/kafka/api/LeaderAndIsrRequest.scala | 12 ++-- .../main/scala/kafka/api/OffsetCommitRequest.scala | 10 +-- .../main/scala/kafka/api/OffsetFetchRequest.scala | 15 ++--- core/src/main/scala/kafka/api/OffsetRequest.scala | 7 ++- .../src/main/scala/kafka/api/ProducerRequest.scala | 7 ++- .../main/scala/kafka/api/StopReplicaRequest.scala | 4 +- .../scala/kafka/api/TopicMetadataRequest.scala | 8 ++- .../scala/kafka/api/UpdateMetadataRequest.scala | 4 +- .../scala/kafka/javaapi/TopicMetadataRequest.scala | 7 +-- .../main/scala/kafka/network/BlockingChannel.scala | 6 +- .../kafka/network/BoundedByteBufferSend.scala | 71 ---------------------- .../main/scala/kafka/network/ByteBufferSend.scala | 40 ------------ core/src/main/scala/kafka/network/Handler.scala | 2 +- .../main/scala/kafka/network/RequestChannel.scala | 1 + .../kafka/network/RequestOrResponseSend.scala | 57 +++++++++++++++++ .../main/scala/kafka/network/SocketServer.scala | 2 +- .../main/scala/kafka/network/Transmission.scala | 40 +++++------- .../main/scala/kafka/producer/SyncProducer.scala | 8 +-- core/src/main/scala/kafka/server/KafkaApis.scala | 37 ++++++----- .../main/scala/kafka/server/MessageSetSend.scala | 17 ++++-- .../unit/kafka/network/SocketServerTest.scala | 3 +- 28 files changed, 251 insertions(+), 225 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java delete mode 100644 core/src/main/scala/kafka/network/BoundedByteBufferSend.scala delete mode 100644 core/src/main/scala/kafka/network/ByteBufferSend.scala create mode 100644 core/src/main/scala/kafka/network/RequestOrResponseSend.scala diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index c8213e1..e828833 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -52,7 +52,7 @@ public class ByteBufferSend implements Send { } @Override - public int remaining() { + public long remaining() { return this.remaining; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java index 68327cd..d1ee959 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java @@ -23,10 +23,16 @@ import java.nio.ByteBuffer; */ public class NetworkSend extends ByteBufferSend { + public final static int UNKNOWN_DEST = -1; + public NetworkSend(int destination, ByteBuffer... buffers) { super(destination, sizeDelimit(buffers)); } + public NetworkSend(ByteBuffer... buffers) { + this(UNKNOWN_DEST, buffers); + } + private static ByteBuffer[] sizeDelimit(ByteBuffer[] buffers) { int size = 0; for (int i = 0; i < buffers.length; i++) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index 5d321a0..234945b 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -29,7 +29,7 @@ public interface Send { /** * The number of bytes remaining to send */ - public int remaining(); + public long remaining(); /** * Is this send complete? diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java new file mode 100644 index 0000000..048119f --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.network.NetworkSend; +import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; + +public class ResponseSend extends NetworkSend { + + private final static int UNKNOWN_DEST = -1; + + public ResponseSend(int destination, ResponseHeader header, Struct body) { + super(destination, serialize(header, body)); + } + + public ResponseSend(ResponseHeader header, AbstractRequestResponse response) { + this(UNKNOWN_DEST, header, response.toStruct()); + } + + private static ByteBuffer serialize(ResponseHeader header, Struct body) { + ByteBuffer buffer = ByteBuffer.allocate(header.sizeOf() + body.sizeOf()); + header.writeTo(buffer); + body.writeTo(buffer); + buffer.rewind(); + return buffer; + } +} diff --git a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala index a3b1b78..05cda52 100644 --- a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala @@ -18,9 +18,10 @@ package kafka.api import java.nio.ByteBuffer -import kafka.network.{BoundedByteBufferSend, RequestChannel} -import kafka.network.RequestChannel.Response + import kafka.common.ErrorMapping +import kafka.network.{RequestOrResponseSend, RequestChannel} +import kafka.network.RequestChannel.Response object ConsumerMetadataRequest { val CurrentVersion = 0.shortValue @@ -64,7 +65,7 @@ case class ConsumerMetadataRequest(group: String, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { // return ConsumerCoordinatorNotAvailable for all uncaught errors val errorResponse = ConsumerMetadataResponse(None, ErrorMapping.ConsumerCoordinatorNotAvailableCode, correlationId) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } def describe(details: Boolean) = { diff --git a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala index 5be393a..ff4ddb9 100644 --- a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala +++ b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala @@ -18,10 +18,9 @@ package kafka.api import java.nio.ByteBuffer -import kafka.api.ApiUtils._ -import collection.mutable.ListBuffer -import kafka.network.{BoundedByteBufferSend, RequestChannel} -import kafka.common.{TopicAndPartition, ErrorMapping} + +import kafka.common.{ErrorMapping, TopicAndPartition} +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response import kafka.utils.Logging @@ -63,7 +62,7 @@ case class ControlledShutdownRequest(val versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = ControlledShutdownResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]]), Set.empty[TopicAndPartition]) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean = false): String = { diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 75aaf57..002b17c 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -22,8 +22,9 @@ import java.nio.channels.GatheringByteChannel import kafka.common.{TopicAndPartition, ErrorMapping} import kafka.message.{MessageSet, ByteBufferMessageSet} -import kafka.network.{MultiSend, Send} +import kafka.network.MultiSend import kafka.api.ApiUtils._ +import org.apache.kafka.common.network.Send import scala.collection._ @@ -62,9 +63,15 @@ class PartitionDataSend(val partitionId: Int, buffer.putInt(partitionData.messages.sizeInBytes) buffer.rewind() - override def complete = !buffer.hasRemaining && messagesSentSize >= messageSize + override def completed = !buffer.hasRemaining && messagesSentSize >= messageSize - override def writeTo(channel: GatheringByteChannel): Int = { + override def destination: Int = -1 + + override def reify = null + + override def remaining: Long = buffer.remaining().toLong + (messageSize - messagesSentSize) + + override def writeTo(channel: GatheringByteChannel): Long = { var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) @@ -104,9 +111,15 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti class TopicDataSend(val topicData: TopicData) extends Send { private val size = topicData.sizeInBytes - private var sent = 0 + private var sent = 0L + + override def completed: Boolean = sent >= size + + override def destination: Int = -1 + + override def reify = null - override def complete = sent >= size + override def remaining: Long = (size.toLong - sent) private val buffer = ByteBuffer.allocate(topicData.headerSize) writeShortString(buffer, topicData.topic) @@ -118,12 +131,12 @@ class TopicDataSend(val topicData: TopicData) extends Send { val expectedBytesToWrite = topicData.sizeInBytes - topicData.headerSize } - def writeTo(channel: GatheringByteChannel): Int = { - expectIncomplete() - var written = 0 + def writeTo(channel: GatheringByteChannel): Long = { + // TODO: expectIncomplete() + var written = 0L if(buffer.hasRemaining) written += channel.write(buffer) - if(!buffer.hasRemaining && !sends.complete) { + if(!buffer.hasRemaining && !sends.completed) { written += sends.writeTo(channel) } sent += written @@ -203,11 +216,17 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val fetchResponse: FetchResponse) extends Send { private val size = fetchResponse.sizeInBytes - private var sent = 0 + private var sent = 0L private val sendSize = 4 /* for size */ + size - override def complete = sent >= sendSize + override def completed = sent >= sendSize + + override def destination = -1 + + override def reify = null + + override def remaining = sendSize - sent private val buffer = ByteBuffer.allocate(4 /* for size */ + FetchResponse.headerSize) buffer.putInt(size) @@ -222,12 +241,12 @@ class FetchResponseSend(val fetchResponse: FetchResponse) extends Send { val expectedBytesToWrite = fetchResponse.sizeInBytes - FetchResponse.headerSize } - def writeTo(channel: GatheringByteChannel):Int = { - expectIncomplete() - var written = 0 + def writeTo(channel: GatheringByteChannel):Long = { + //TODO: expectIncomplete() + var written = 0L if(buffer.hasRemaining) written += channel.write(buffer) - if(!buffer.hasRemaining && !sends.complete) { + if(!buffer.hasRemaining && !sends.completed) { written += sends.writeTo(channel) } sent += written diff --git a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala index 2fad585..e03bdb0 100644 --- a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala +++ b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala @@ -19,14 +19,16 @@ package kafka.api import java.nio._ -import kafka.utils._ + import kafka.api.ApiUtils._ import kafka.cluster.BrokerEndPoint -import kafka.controller.LeaderIsrAndControllerEpoch -import kafka.network.{BoundedByteBufferSend, RequestChannel} import kafka.common.ErrorMapping +import kafka.controller.LeaderIsrAndControllerEpoch +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response -import collection.Set +import kafka.utils._ + +import scala.collection.Set object LeaderAndIsr { @@ -184,7 +186,7 @@ case class LeaderAndIsrRequest (versionId: Short, case (topicAndPartition, partitionAndState) => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = LeaderAndIsrResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala index cf8e6ac..226b8f8 100644 --- a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala @@ -18,11 +18,13 @@ package kafka.api import java.nio.ByteBuffer + import kafka.api.ApiUtils._ -import kafka.utils.{SystemTime, Logging} -import kafka.network.{RequestChannel, BoundedByteBufferSend} -import kafka.common.{OffsetMetadata, OffsetAndMetadata, ErrorMapping, TopicAndPartition} +import kafka.common.{ErrorMapping, OffsetAndMetadata, TopicAndPartition} +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response +import kafka.utils.Logging + import scala.collection._ object OffsetCommitRequest extends Logging { @@ -161,7 +163,7 @@ case class OffsetCommitRequest(groupId: String, val commitStatus = requestInfo.mapValues(_ => errorCode) val commitResponse = OffsetCommitResponse(commitStatus, correlationId) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(commitResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(commitResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala index 67811a7..f179903 100644 --- a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala @@ -17,16 +17,13 @@ package kafka.api +import java.nio.ByteBuffer + import kafka.api.ApiUtils._ -import kafka.utils.Logging -import kafka.network.{BoundedByteBufferSend, RequestChannel} -import kafka.common._ -import kafka.common.TopicAndPartition +import kafka.common.{TopicAndPartition, _} +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response - -import scala.Some - -import java.nio.ByteBuffer +import kafka.utils.Logging object OffsetFetchRequest extends Logging { val CurrentVersion: Short = 0 @@ -99,7 +96,7 @@ case class OffsetFetchRequest(groupId: String, )) }.toMap val errorResponse = OffsetFetchResponse(requestInfo=responseMap, correlationId=correlationId) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetRequest.scala b/core/src/main/scala/kafka/api/OffsetRequest.scala index 3d483bc..b1bf46a 100644 --- a/core/src/main/scala/kafka/api/OffsetRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetRequest.scala @@ -18,9 +18,10 @@ package kafka.api import java.nio.ByteBuffer -import kafka.common.{ErrorMapping, TopicAndPartition} + import kafka.api.ApiUtils._ -import kafka.network.{BoundedByteBufferSend, RequestChannel} +import kafka.common.{ErrorMapping, TopicAndPartition} +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response @@ -117,7 +118,7 @@ case class OffsetRequest(requestInfo: Map[TopicAndPartition, PartitionOffsetRequ (topicAndPartition, PartitionOffsetsResponse(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), null)) } val errorResponse = OffsetResponse(correlationId, partitionOffsetResponseMap) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/ProducerRequest.scala b/core/src/main/scala/kafka/api/ProducerRequest.scala index 570b2da..f5167e4 100644 --- a/core/src/main/scala/kafka/api/ProducerRequest.scala +++ b/core/src/main/scala/kafka/api/ProducerRequest.scala @@ -18,11 +18,12 @@ package kafka.api import java.nio._ -import kafka.message._ + import kafka.api.ApiUtils._ import kafka.common._ +import kafka.message._ +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response -import kafka.network.{RequestChannel, BoundedByteBufferSend} object ProducerRequest { val CurrentVersion = 0.shortValue @@ -136,7 +137,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, (topicAndPartition, ProducerResponseStatus(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1l)) } val errorResponse = ProducerResponse(correlationId, producerResponseStatus) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } } diff --git a/core/src/main/scala/kafka/api/StopReplicaRequest.scala b/core/src/main/scala/kafka/api/StopReplicaRequest.scala index 5e14987..aee6973 100644 --- a/core/src/main/scala/kafka/api/StopReplicaRequest.scala +++ b/core/src/main/scala/kafka/api/StopReplicaRequest.scala @@ -19,7 +19,7 @@ package kafka.api import java.nio._ import kafka.api.ApiUtils._ -import kafka.network.{BoundedByteBufferSend, RequestChannel, InvalidRequestException} +import kafka.network.{RequestOrResponseSend, RequestChannel, InvalidRequestException} import kafka.common.{TopicAndPartition, ErrorMapping} import kafka.network.RequestChannel.Response import kafka.utils.Logging @@ -106,7 +106,7 @@ case class StopReplicaRequest(versionId: Short, case topicAndPartition => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) }.toMap val errorResponse = StopReplicaResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala index 7dca09c..3cdbd40 100644 --- a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala @@ -18,13 +18,15 @@ package kafka.api import java.nio.ByteBuffer + import kafka.api.ApiUtils._ -import collection.mutable.ListBuffer -import kafka.network.{BoundedByteBufferSend, RequestChannel} import kafka.common.ErrorMapping +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response import kafka.utils.Logging +import scala.collection.mutable.ListBuffer + object TopicMetadataRequest extends Logging { val CurrentVersion = 0.shortValue val DefaultClientId = "" @@ -80,7 +82,7 @@ case class TopicMetadataRequest(val versionId: Short, topic => TopicMetadata(topic, Nil, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = TopicMetadataResponse(Seq(), topicMetadata, correlationId) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala index 69f0397..e463395 100644 --- a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala @@ -21,8 +21,8 @@ import java.nio.ByteBuffer import kafka.api.ApiUtils._ import kafka.cluster.{Broker, BrokerEndPoint} import kafka.common.{ErrorMapping, KafkaException, TopicAndPartition} +import kafka.network.{RequestOrResponseSend, RequestChannel} import kafka.network.RequestChannel.Response -import kafka.network.{BoundedByteBufferSend, RequestChannel} import org.apache.kafka.common.protocol.SecurityProtocol import scala.collection.Set @@ -128,7 +128,7 @@ case class UpdateMetadataRequest (versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = new UpdateMetadataResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]])) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala b/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala index b0b7be1..568d0ac 100644 --- a/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/javaapi/TopicMetadataRequest.scala @@ -16,12 +16,11 @@ */ package kafka.javaapi -import kafka.api._ import java.nio.ByteBuffer + +import kafka.api._ + import scala.collection.mutable -import kafka.network.{BoundedByteBufferSend, RequestChannel} -import kafka.common.ErrorMapping -import kafka.network.RequestChannel.Response class TopicMetadataRequest(val versionId: Short, val correlationId: Int, diff --git a/core/src/main/scala/kafka/network/BlockingChannel.scala b/core/src/main/scala/kafka/network/BlockingChannel.scala index 7c0d8fc..7415f36 100644 --- a/core/src/main/scala/kafka/network/BlockingChannel.scala +++ b/core/src/main/scala/kafka/network/BlockingChannel.scala @@ -21,7 +21,7 @@ import java.net.InetSocketAddress import java.nio.channels._ import kafka.utils.{nonthreadsafe, Logging} import kafka.api.RequestOrResponse -import org.apache.kafka.common.network.NetworkReceive +import org.apache.kafka.common.network.{NetworkSend, NetworkReceive} object BlockingChannel{ @@ -96,11 +96,11 @@ class BlockingChannel( val host: String, def isConnected = connected - def send(request: RequestOrResponse):Int = { + def send(request: RequestOrResponse): Long = { if(!connected) throw new ClosedChannelException() - val send = new BoundedByteBufferSend(request) + val send = new RequestOrResponseSend(request) send.writeCompletely(writeChannel) } diff --git a/core/src/main/scala/kafka/network/BoundedByteBufferSend.scala b/core/src/main/scala/kafka/network/BoundedByteBufferSend.scala deleted file mode 100644 index b95b73b..0000000 --- a/core/src/main/scala/kafka/network/BoundedByteBufferSend.scala +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.network - -import java.nio._ -import java.nio.channels._ -import kafka.utils._ -import kafka.api.RequestOrResponse -import org.apache.kafka.common.requests.{AbstractRequestResponse, ResponseHeader} - -@nonthreadsafe -private[kafka] class BoundedByteBufferSend(val buffer: ByteBuffer) extends Send { - - private val sizeBuffer = ByteBuffer.allocate(4) - - // Avoid possibility of overflow for 2GB-4 byte buffer - if(buffer.remaining > Int.MaxValue - sizeBuffer.limit) - throw new IllegalStateException("Attempt to create a bounded buffer of " + buffer.remaining + " bytes, but the maximum " + - "allowable size for a bounded buffer is " + (Int.MaxValue - sizeBuffer.limit) + ".") - sizeBuffer.putInt(buffer.limit) - sizeBuffer.rewind() - - var complete: Boolean = false - - def this(size: Int) = this(ByteBuffer.allocate(size)) - - def this(request: RequestOrResponse) = { - this(request.sizeInBytes + (if(request.requestId != None) 2 else 0)) - request.requestId match { - case Some(requestId) => - buffer.putShort(requestId) - case None => - } - - request.writeTo(buffer) - buffer.rewind() - } - - def this(header: ResponseHeader, body: AbstractRequestResponse) = { - this(header.sizeOf + body.sizeOf) - header.writeTo(buffer) - body.writeTo(buffer) - buffer.rewind - } - - - def writeTo(channel: GatheringByteChannel): Int = { - expectIncomplete() - val written = channel.write(Array(sizeBuffer, buffer)) - // if we are done, mark it off - if(!buffer.hasRemaining) - complete = true - written.asInstanceOf[Int] - } - -} diff --git a/core/src/main/scala/kafka/network/ByteBufferSend.scala b/core/src/main/scala/kafka/network/ByteBufferSend.scala deleted file mode 100644 index af30042..0000000 --- a/core/src/main/scala/kafka/network/ByteBufferSend.scala +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.network - -import java.nio._ -import java.nio.channels._ -import kafka.utils._ - -@nonthreadsafe -private[kafka] class ByteBufferSend(val buffer: ByteBuffer) extends Send { - - var complete: Boolean = false - - def this(size: Int) = this(ByteBuffer.allocate(size)) - - def writeTo(channel: GatheringByteChannel): Int = { - expectIncomplete() - var written = 0 - written += channel.write(buffer) - if(!buffer.hasRemaining) - complete = true - written - } - -} diff --git a/core/src/main/scala/kafka/network/Handler.scala b/core/src/main/scala/kafka/network/Handler.scala index cd1d143..2148e0c 100644 --- a/core/src/main/scala/kafka/network/Handler.scala +++ b/core/src/main/scala/kafka/network/Handler.scala @@ -17,7 +17,7 @@ package kafka.network -import org.apache.kafka.common.network.NetworkReceive +import org.apache.kafka.common.network.{Send, NetworkReceive} private[kafka] object Handler { diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 1d9c57b..47d9a5d 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -26,6 +26,7 @@ import kafka.common.TopicAndPartition import kafka.utils.{Logging, SystemTime} import kafka.message.ByteBufferMessageSet import java.net._ +import org.apache.kafka.common.network.Send import org.apache.kafka.common.protocol.SecurityProtocol import org.apache.kafka.common.requests.{AbstractRequest, RequestHeader} import org.apache.log4j.Logger diff --git a/core/src/main/scala/kafka/network/RequestOrResponseSend.scala b/core/src/main/scala/kafka/network/RequestOrResponseSend.scala new file mode 100644 index 0000000..424258f --- /dev/null +++ b/core/src/main/scala/kafka/network/RequestOrResponseSend.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.network + +import java.nio.ByteBuffer +import java.nio.channels.GatheringByteChannel + +import kafka.api.RequestOrResponse +import kafka.utils.Logging +import org.apache.kafka.common.network.NetworkSend + +object RequestOrResponseSend { + def serialize(request: RequestOrResponse): ByteBuffer = { + val buffer = ByteBuffer.allocate(request.sizeInBytes + (if(request.requestId != None) 2 else 0)) + request.requestId match { + case Some(requestId) => + buffer.putShort(requestId) + case None => + } + request.writeTo(buffer) + buffer.rewind() + buffer + } +} + +class RequestOrResponseSend(val buffer: ByteBuffer) extends NetworkSend(buffer) with Logging { + + def this(request: RequestOrResponse) { + this(RequestOrResponseSend.serialize(request)) + } + + def writeCompletely(channel: GatheringByteChannel): Long = { + var totalWritten = 0L + while(!completed()) { + val written = writeTo(channel) + trace(written + " bytes written.") + totalWritten += written + } + totalWritten + } + +} diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 3998eb4..28bd4c3 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -513,7 +513,7 @@ private[kafka] class Processor(val id: Int, throw new IllegalStateException("Registered for write interest but no response attached to key.") val written = responseSend.writeTo(socketChannel) trace(written + " bytes written to " + socketChannel.socket.getRemoteSocketAddress() + " using key " + key) - if(responseSend.complete) { + if(responseSend.completed()) { response.request.updateRequestMetrics() key.attach(null) trace("Finished writing, registering for read on connection " + socketChannel.socket.getRemoteSocketAddress()) diff --git a/core/src/main/scala/kafka/network/Transmission.scala b/core/src/main/scala/kafka/network/Transmission.scala index 296a78b..4c7a557 100644 --- a/core/src/main/scala/kafka/network/Transmission.scala +++ b/core/src/main/scala/kafka/network/Transmission.scala @@ -21,6 +21,7 @@ import java.nio._ import java.nio.channels._ import kafka.utils.Logging import kafka.common.KafkaException +import org.apache.kafka.common.network.Send /** * Represents a stateful transfer of data to or from the network @@ -43,55 +44,44 @@ private[network] trait Transmission extends Logging { -/** - * A transmission that is being sent out to the channel - */ -trait Send extends Transmission { - - def writeTo(channel: GatheringByteChannel): Int - def writeCompletely(channel: GatheringByteChannel): Int = { - var totalWritten = 0 - while(!complete) { - val written = writeTo(channel) - trace(written + " bytes written.") - totalWritten += written - } - totalWritten - } - -} /** * A set of composite sends, sent one after another */ -abstract class MultiSend[S <: Send](val sends: List[S]) extends Send { +abstract class MultiSend[S <: Send](val sends: List[S]) extends Send with Logging { val expectedBytesToWrite: Int private var current = sends - var totalWritten = 0 + var totalWritten = 0L + + override def destination = -1; + + override def reify = null; + + override def remaining = (expectedBytesToWrite - totalWritten) /** * This method continues to write to the socket buffer till an incomplete * write happens. On an incomplete write, it returns to the caller to give it * a chance to schedule other work till the buffered write completes. */ - def writeTo(channel: GatheringByteChannel): Int = { - expectIncomplete - var totalWrittenPerCall = 0 + def writeTo(channel: GatheringByteChannel): Long = { + //TODO: Is this needed? expectIncomplete + var totalWrittenPerCall = 0L var sendComplete: Boolean = false do { val written = current.head.writeTo(channel) totalWritten += written totalWrittenPerCall += written - sendComplete = current.head.complete + sendComplete = current.head.completed() if(sendComplete) current = current.tail - } while (!complete && sendComplete) + } while (!completed && sendComplete) trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + expectedBytesToWrite) totalWrittenPerCall } - def complete: Boolean = { + def completed: Boolean = { if (current == Nil) { if (totalWritten != expectedBytesToWrite) error("mismatch in sending bytes over socket; expected: " + expectedBytesToWrite + " actual: " + totalWritten) diff --git a/core/src/main/scala/kafka/producer/SyncProducer.scala b/core/src/main/scala/kafka/producer/SyncProducer.scala index a19058e..2a6c91e 100644 --- a/core/src/main/scala/kafka/producer/SyncProducer.scala +++ b/core/src/main/scala/kafka/producer/SyncProducer.scala @@ -17,11 +17,11 @@ package kafka.producer -import kafka.api._ -import kafka.network.{BlockingChannel, BoundedByteBufferSend} -import kafka.utils._ import java.util.Random +import kafka.api._ +import kafka.network.{RequestOrResponseSend, BlockingChannel} +import kafka.utils._ import org.apache.kafka.common.network.NetworkReceive import org.apache.kafka.common.utils.Utils._ @@ -51,7 +51,7 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { * data. So, leaving the rest of the logging at TRACE, while errors should be logged at ERROR level */ if (logger.isDebugEnabled) { - val buffer = new BoundedByteBufferSend(request).buffer + val buffer = new RequestOrResponseSend(request).buffer trace("verifying sendbuffer of size " + buffer.limit) val requestTypeId = buffer.getShort() if(requestTypeId == RequestKeys.ProduceKey) { diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index b4004aa..feea236 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,8 +17,16 @@ package kafka.server + +import kafka.api.ConsumerMetadataRequest +import kafka.api.ConsumerMetadataResponse +import kafka.api.FetchRequest +import kafka.api.FetchResponse +import kafka.api.OffsetCommitRequest +import kafka.api.OffsetCommitResponse +import kafka.api.OffsetFetchRequest +import kafka.api.OffsetFetchResponse import org.apache.kafka.common.protocol.SecurityProtocol -import org.apache.kafka.common.requests.{JoinGroupResponse, JoinGroupRequest, HeartbeatRequest, HeartbeatResponse, ResponseHeader} import org.apache.kafka.common.TopicPartition import kafka.api._ import kafka.admin.AdminUtils @@ -29,6 +37,7 @@ import kafka.log._ import kafka.network._ import kafka.network.RequestChannel.Response import kafka.utils.{SystemTime, Logging} +import org.apache.kafka.common.requests.{JoinGroupRequest, JoinGroupResponse, HeartbeatRequest, HeartbeatResponse, ResponseHeader, ResponseSend} import scala.collection._ @@ -84,7 +93,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (response == null) requestChannel.closeConnection(request.processor, request) else - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(respHeader, response))) + requestChannel.sendResponse(new Response(request, new ResponseSend(respHeader, response))) } error("error when handling request %s".format(request.requestObj), e) } finally @@ -99,7 +108,7 @@ class KafkaApis(val requestChannel: RequestChannel, try { val (response, error) = replicaManager.becomeLeaderOrFollower(leaderAndIsrRequest, offsetManager) val leaderAndIsrResponse = new LeaderAndIsrResponse(leaderAndIsrRequest.correlationId, response, error) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(leaderAndIsrResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(leaderAndIsrResponse))) } catch { case e: KafkaStorageException => fatal("Disk error during leadership change.", e) @@ -114,7 +123,7 @@ class KafkaApis(val requestChannel: RequestChannel, val stopReplicaRequest = request.requestObj.asInstanceOf[StopReplicaRequest] val (response, error) = replicaManager.stopReplicas(stopReplicaRequest) val stopReplicaResponse = new StopReplicaResponse(stopReplicaRequest.correlationId, response.toMap, error) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(stopReplicaResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(stopReplicaResponse))) replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads() } @@ -123,7 +132,7 @@ class KafkaApis(val requestChannel: RequestChannel, replicaManager.maybeUpdateMetadataCache(updateMetadataRequest, metadataCache) val updateMetadataResponse = new UpdateMetadataResponse(updateMetadataRequest.correlationId) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(updateMetadataResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(updateMetadataResponse))) } def handleControlledShutdownRequest(request: RequestChannel.Request) { @@ -134,7 +143,7 @@ class KafkaApis(val requestChannel: RequestChannel, val partitionsRemaining = controller.shutdownBroker(controlledShutdownRequest.brokerId) val controlledShutdownResponse = new ControlledShutdownResponse(controlledShutdownRequest.correlationId, ErrorMapping.NoError, partitionsRemaining) - requestChannel.sendResponse(new Response(request, new BoundedByteBufferSend(controlledShutdownResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(controlledShutdownResponse))) } @@ -158,7 +167,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val response = OffsetCommitResponse(commitStatus, offsetCommitRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } // compute the retention time based on the request version: @@ -234,7 +243,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } else { val response = ProducerResponse(produceRequest.correlationId, responseStatus) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } } @@ -337,7 +346,7 @@ class KafkaApis(val requestChannel: RequestChannel, } }) val response = OffsetResponse(offsetRequest.correlationId, responseMap) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } def fetchOffsets(logManager: LogManager, topicAndPartition: TopicAndPartition, timestamp: Long, maxNumOffsets: Int): Seq[Long] = { @@ -440,7 +449,7 @@ class KafkaApis(val requestChannel: RequestChannel, val brokers = metadataCache.getAliveBrokers trace("Sending topic metadata %s and brokers %s for correlation id %d to client %s".format(topicMetadata.mkString(","), brokers.mkString(","), metadataRequest.correlationId, metadataRequest.clientId)) val response = new TopicMetadataResponse(brokers.map(_.getBrokerEndPoint(request.securityProtocol)), topicMetadata, metadataRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } /* @@ -464,7 +473,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending offset fetch response %s for correlation id %d to client %s." .format(response, offsetFetchRequest.correlationId, offsetFetchRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } /* @@ -489,7 +498,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending consumer metadata %s for correlation id %d to client %s." .format(response, consumerMetadataRequest.correlationId, consumerMetadataRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) } def handleJoinGroupRequest(request: RequestChannel.Request) { @@ -502,7 +511,7 @@ class KafkaApis(val requestChannel: RequestChannel, def sendResponseCallback(partitions: List[TopicAndPartition], generationId: Int, errorCode: Short) { val partitionList = partitions.map(tp => new TopicPartition(tp.topic, tp.partition)).toBuffer val responseBody = new JoinGroupResponse(errorCode, generationId, joinGroupRequest.consumerId, partitionList) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(respHeader, responseBody))) + requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(respHeader, responseBody))) } // let the coordinator to handle join-group @@ -522,7 +531,7 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for sending a heartbeat response def sendResponseCallback(errorCode: Short) { val response = new HeartbeatResponse(errorCode) - requestChannel.sendResponse(new RequestChannel.Response(request, new BoundedByteBufferSend(respHeader, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(respHeader, response))) } // let the coordinator to handle heartbeat diff --git a/core/src/main/scala/kafka/server/MessageSetSend.scala b/core/src/main/scala/kafka/server/MessageSetSend.scala index 5667648..484c5d8 100644 --- a/core/src/main/scala/kafka/server/MessageSetSend.scala +++ b/core/src/main/scala/kafka/server/MessageSetSend.scala @@ -23,13 +23,14 @@ import kafka.network._ import kafka.message._ import kafka.utils._ import kafka.common.ErrorMapping +import org.apache.kafka.common.network.Send /** * A zero-copy message response that writes the bytes needed directly from the file * wholly in kernel space */ @nonthreadsafe -private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Short) extends Send { +private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Short) extends Send with Logging{ private var sent: Int = 0 private val size: Int = messages.sizeInBytes @@ -38,14 +39,20 @@ private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Sh header.putShort(errorCode) header.rewind() - var complete: Boolean = false + var completed: Boolean = false def this(messages: MessageSet) = this(messages, ErrorMapping.NoError) def this() = this(MessageSet.Empty) - def writeTo(channel: GatheringByteChannel): Int = { - expectIncomplete() + override def destination = -1 + + override def reify = null; + + override def remaining = size - sent + + def writeTo(channel: GatheringByteChannel): Long = { + //TODO: expectIncomplete() var written = 0 if(header.hasRemaining) written += channel.write(header) @@ -62,7 +69,7 @@ private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Sh } if(sent >= size) - complete = true + completed = true written } diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 95d5621..6fa4f12 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -20,6 +20,7 @@ package kafka.network; import java.net._ import java.io._ import kafka.cluster.EndPoint +import org.apache.kafka.common.network.NetworkSend import org.apache.kafka.common.protocol.SecurityProtocol import org.junit._ import org.scalatest.junit.JUnitSuite @@ -71,7 +72,7 @@ class SocketServerTest extends JUnitSuite { val byteBuffer = ByteBuffer.allocate(request.requestObj.sizeInBytes) request.requestObj.writeTo(byteBuffer) byteBuffer.rewind() - val send = new BoundedByteBufferSend(byteBuffer) + val send = new NetworkSend(byteBuffer) channel.sendResponse(new RequestChannel.Response(request.processor, request, send)) } -- 2.3.2 (Apple Git-55) From 12fff39132fbd45e120645400f3b311fcec49475 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 9 Apr 2015 20:44:27 -0700 Subject: [PATCH 03/24] implement maxSize and improved docs --- .../main/java/org/apache/kafka/common/network/NetworkReceive.java | 7 ++++++- .../main/java/org/apache/kafka/common/requests/ResponseSend.java | 2 -- core/src/main/scala/kafka/network/BlockingChannel.scala | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index 5207807..cf97ea5 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -72,13 +72,15 @@ public class NetworkReceive implements Receive { return new ByteBuffer[] {this.buffer}; } - @Override + public long readFrom(ScatteringByteChannel channel) throws IOException { return readFromReadableChannel(channel); } // Need a method to read from ReadableByteChannel because BlockingChannel requires read with timeout // See: http://stackoverflow.com/questions/2866557/timeout-for-socketchannel-doesnt-work + // This can go away after we get rid of BlockingChannel + @Deprecated public long readFromReadableChannel(ReadableByteChannel channel) throws IOException { int read = 0; if (size.hasRemaining()) { @@ -91,6 +93,8 @@ public class NetworkReceive implements Receive { int requestSize = size.getInt(); if (requestSize < 0) throw new IllegalStateException("Invalid request (size = " + requestSize + ")"); + if (maxSize != UNLIMITED && requestSize > maxSize) + throw new IllegalStateException("Invalid request (size = " + requestSize + " larger than " + maxSize + ")"); this.buffer = ByteBuffer.allocate(requestSize); } } @@ -109,6 +113,7 @@ public class NetworkReceive implements Receive { } // Used only by BlockingChannel, so we may be able to get rid of this when/if we get rid of BlockingChannel + @Deprecated public long readCompletely(ReadableByteChannel channel) throws IOException { int totalRead = 0; while (!complete()) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java index 048119f..c510559 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java @@ -23,8 +23,6 @@ import java.nio.ByteBuffer; public class ResponseSend extends NetworkSend { - private final static int UNKNOWN_DEST = -1; - public ResponseSend(int destination, ResponseHeader header, Struct body) { super(destination, serialize(header, body)); } diff --git a/core/src/main/scala/kafka/network/BlockingChannel.scala b/core/src/main/scala/kafka/network/BlockingChannel.scala index 7415f36..45adb92 100644 --- a/core/src/main/scala/kafka/network/BlockingChannel.scala +++ b/core/src/main/scala/kafka/network/BlockingChannel.scala @@ -60,6 +60,8 @@ class BlockingChannel( val host: String, channel.socket.connect(new InetSocketAddress(host, port), connectTimeoutMs) writeChannel = channel + // Need to create a new ReadableByteChannel from input stream because SocketChannel doesn't implement read with timeout + // See: http://stackoverflow.com/questions/2866557/timeout-for-socketchannel-doesnt-work readChannel = Channels.newChannel(channel.socket().getInputStream) connected = true // settings may not match what we requested above -- 2.3.2 (Apple Git-55) From cac598b205a828e8da28cc4779eb2de4b6492082 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Tue, 14 Apr 2015 13:15:20 -0700 Subject: [PATCH 04/24] moved selector out of abstract thread --- .../org/apache/kafka/common/network/Selector.java | 55 +++++++------ .../main/scala/kafka/network/SocketServer.scala | 91 ++++++++++++---------- 2 files changed, 83 insertions(+), 63 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 57de058..2051261 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -17,10 +17,7 @@ import java.io.IOException; import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.Socket; -import java.nio.channels.CancelledKeyException; -import java.nio.channels.SelectionKey; -import java.nio.channels.SocketChannel; -import java.nio.channels.UnresolvedAddressException; +import java.nio.channels.*; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -45,15 +42,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * A selector interface for doing non-blocking multi-connection network I/O. + * A nioSelector interface for doing non-blocking multi-connection network I/O. *

* This class works with {@link NetworkSend} and {@link NetworkReceive} to transmit size-delimited network requests and * responses. *

- * A connection can be added to the selector associated with an integer id by doing + * A connection can be added to the nioSelector associated with an integer id by doing * * 

- * selector.connect(42, new InetSocketAddress("google.com", server.port), 64000, 64000);
+ * nioSelector.connect(42, new InetSocketAddress("google.com", server.port), 64000, 64000);
  *
* * The connect call does not block on the creation of the TCP connection, so the connect method only begins initiating @@ -64,10 +61,10 @@ import org.slf4j.LoggerFactory; * * 
  * List<NetworkRequest> requestsToSend = Arrays.asList(new NetworkRequest(0, myBytes), new NetworkRequest(1, myOtherBytes));
- * selector.poll(TIMEOUT_MS, requestsToSend);
+ * nioSelector.poll(TIMEOUT_MS, requestsToSend);
  *
* - * The selector maintains several lists that are reset by each call to poll() which are available via + * The nioSelector maintains several lists that are reset by each call to poll() which are available via * various getters. These are reset by each call to poll(). * * This class is not thread safe! @@ -76,7 +73,7 @@ public class Selector implements Selectable { private static final Logger log = LoggerFactory.getLogger(Selector.class); - private final java.nio.channels.Selector selector; + private final java.nio.channels.Selector nioSelector; private final Map keys; private final List completedSends; private final List completedReceives; @@ -89,11 +86,11 @@ public class Selector implements Selectable { private final Map metricTags; /** - * Create a new selector + * Create a new nioSelector */ public Selector(Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { try { - this.selector = java.nio.channels.Selector.open(); + this.nioSelector = java.nio.channels.Selector.open(); } catch (IOException e) { throw new KafkaException(e); } @@ -110,7 +107,7 @@ public class Selector implements Selectable { } /** - * Begin connecting to the given address and add the connection to this selector associated with the given id + * Begin connecting to the given address and add the connection to this nioSelector associated with the given id * number. *

* Note that this call only initiates the connection, which will be completed on a future {@link #poll(long, List)} @@ -143,12 +140,22 @@ public class Selector implements Selectable { channel.close(); throw e; } - SelectionKey key = channel.register(this.selector, SelectionKey.OP_CONNECT); + SelectionKey key = channel.register(this.nioSelector, SelectionKey.OP_CONNECT); key.attach(new Transmissions(id)); this.keys.put(id, key); } /** + * Register the nioSelector with an existing channel + * Use this on server-side, when a connection is accepted by a different thread but processed by the Selector + */ + public void register(int id, SocketChannel channel) throws ClosedChannelException { + if (log.isDebugEnabled()) + log.debug("Processor " + id + " listening to new connection from " + channel.socket().getRemoteSocketAddress()); + channel.register(nioSelector, SelectionKey.OP_READ); + } + + /** * Disconnect any connections for the given id (if there are any). The disconnection is asynchronous and will not be * processed until the next {@link #poll(long, List) poll()} call. */ @@ -160,24 +167,24 @@ public class Selector implements Selectable { } /** - * Interrupt the selector if it is blocked waiting to do I/O. + * Interrupt the nioSelector if it is blocked waiting to do I/O. */ @Override public void wakeup() { - this.selector.wakeup(); + this.nioSelector.wakeup(); } /** - * Close this selector and all associated connections + * Close this nioSelector and all associated connections */ @Override public void close() { - for (SelectionKey key : this.selector.keys()) + for (SelectionKey key : this.nioSelector.keys()) close(key); try { - this.selector.close(); + this.nioSelector.close(); } catch (IOException e) { - log.error("Exception closing selector:", e); + log.error("Exception closing nioSelector:", e); } } @@ -223,7 +230,7 @@ public class Selector implements Selectable { this.sensors.selectTime.record(endSelect - startSelect, time.milliseconds()); if (readyKeys > 0) { - Set keys = this.selector.selectedKeys(); + Set keys = this.nioSelector.selectedKeys(); Iterator iter = keys.iterator(); while (iter.hasNext()) { SelectionKey key = iter.next(); @@ -369,11 +376,11 @@ public class Selector implements Selectable { */ private int select(long ms) throws IOException { if (ms == 0L) - return this.selector.selectNow(); + return this.nioSelector.selectNow(); else if (ms < 0L) - return this.selector.select(); + return this.nioSelector.select(); else - return this.selector.select(ms); + return this.nioSelector.select(ms); } /** diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 28bd4c3..4f1b232 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -25,8 +25,10 @@ import java.io._ import java.nio.channels._ import kafka.cluster.EndPoint +import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.NetworkReceive import org.apache.kafka.common.protocol.SecurityProtocol +import org.apache.kafka.common.utils.{Time, SystemTime} import scala.collection._ @@ -53,7 +55,7 @@ class SocketServer(val brokerId: Int, val connectionsMaxIdleMs: Long, val maxConnectionsPerIpOverrides: Map[String, Int] ) extends Logging with KafkaMetricsGroup { this.logIdent = "[Socket Server on Broker " + brokerId + "], " - private val time = SystemTime + private val time = new SystemTime() private val processors = new Array[Processor](numProcessorThreads) private[network] var acceptors = mutable.Map[EndPoint,Acceptor]() val requestChannel = new RequestChannel(numProcessorThreads, maxQueuedRequests) @@ -92,9 +94,12 @@ class SocketServer(val brokerId: Int, } } + + /* TODO: Do we still need this when Selector has its own metrics? newGauge("ResponsesBeingSent", new Gauge[Int] { def value = processors.foldLeft(0) { (total, p) => total + p.countInterestOps(SelectionKey.OP_WRITE) } }) + */ // register the processor threads for notification of responses requestChannel.addResponseListener((id:Int) => processors(id).wakeup()) @@ -141,17 +146,18 @@ class SocketServer(val brokerId: Int, */ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQuotas) extends Runnable with Logging { - protected val selector = Selector.open() private val startupLatch = new CountDownLatch(1) private val shutdownLatch = new CountDownLatch(1) private val alive = new AtomicBoolean(true) + def shutdownHook + /** * Initiates a graceful shutdown by signaling to stop and waiting for the shutdown to complete */ def shutdown(): Unit = { alive.set(false) - selector.wakeup() + shutdownHook shutdownLatch.await } @@ -178,11 +184,6 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ protected def isRunning = alive.get /** - * Wakeup the thread for selection. - */ - def wakeup() = selector.wakeup() - - /** * Close the given key and associated socket */ def close(key: SelectionKey) { @@ -201,30 +202,6 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ swallowError(channel.close()) } } - - /** - * Close all open connections - */ - def closeAll() { - // removes cancelled keys from selector.keys set - this.selector.selectNow() - val iter = this.selector.keys().iterator() - while (iter.hasNext) { - val key = iter.next() - close(key) - } - } - - def countInterestOps(ops: Int): Int = { - var count = 0 - val it = this.selector.keys().iterator() - while (it.hasNext) { - if ((it.next().interestOps() & ops) != 0) { - count += 1 - } - } - count - } } /** @@ -238,6 +215,7 @@ private[kafka] class Acceptor(val host: String, connectionQuotas: ConnectionQuotas, protocol: SecurityProtocol, portToProtocol: ConcurrentHashMap[Int, SecurityProtocol]) extends AbstractServerThread(connectionQuotas) { + val nioSelector = java.nio.channels.Selector.open() val serverChannel = openServerSocket(host, port) portToProtocol.put(serverChannel.socket().getLocalPort, protocol) @@ -245,13 +223,13 @@ private[kafka] class Acceptor(val host: String, * Accept loop that checks for new connection attempts */ def run() { - serverChannel.register(selector, SelectionKey.OP_ACCEPT); + serverChannel.register(nioSelector, SelectionKey.OP_ACCEPT); startupComplete() var currentProcessor = 0 while(isRunning) { - val ready = selector.select(500) + val ready = nioSelector.select(500) if(ready > 0) { - val keys = selector.selectedKeys() + val keys = nioSelector.selectedKeys() val iter = keys.iterator() while(iter.hasNext && isRunning) { var key: SelectionKey = null @@ -273,7 +251,7 @@ private[kafka] class Acceptor(val host: String, } debug("Closing server socket and selector.") swallowError(serverChannel.close()) - swallowError(selector.close()) + swallowError(nioSelector.close()) shutdownComplete() } @@ -325,6 +303,12 @@ private[kafka] class Acceptor(val host: String, } } + /** + * Wakeup the thread for selection. + */ + def wakeup() = nioSelector.wakeup() + + def shutdownHook = wakeup() } /** @@ -344,9 +328,18 @@ private[kafka] class Processor(val id: Int, private val newConnections = new ConcurrentLinkedQueue[SocketChannel]() private val connectionsMaxIdleNanos = connectionsMaxIdleMs * 1000 * 1000 - private var currentTimeNanos = SystemTime.nanoseconds + private var currentTimeNanos = new SystemTime().nanoseconds private val lruConnections = new util.LinkedHashMap[SelectionKey, Long] private var nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos + private val metrics = new Metrics(time); + private var metricTags = new util.HashMap[String, String]() + metricTags.put("processor-id", id.toString) + private val selector = new org.apache.kafka.common.network.Selector( + metrics, + time, + "SocketServer", + metricTags + ) override def run() { startupComplete() @@ -355,9 +348,9 @@ private[kafka] class Processor(val id: Int, configureNewConnections() // register any new responses for writing processNewResponses() - val startSelectTime = SystemTime.nanoseconds + val startSelectTime = new SystemTime().nanoseconds val ready = selector.select(300) - currentTimeNanos = SystemTime.nanoseconds + currentTimeNanos = new SystemTime().nanoseconds val idleTime = currentTimeNanos - startSelectTime idleMeter.mark(idleTime) // We use a single meter for aggregate idle percentage for the thread pool. @@ -545,6 +538,26 @@ private[kafka] class Processor(val id: Int, } } + /** + * Close all open connections + */ + def closeAll() { + // removes cancelled keys from selector.keys set + this.selector.selectNow() + val iter = this.selector.keys().iterator() + while (iter.hasNext) { + val key = iter.next() + close(key) + } + } + + /** + * Wakeup the thread for selection. + */ + def wakeup() = selector.wakeup() + + def shutdownHook = wakeup + } class ConnectionQuotas(val defaultMax: Int, overrideQuotas: Map[String, Int]) { -- 2.3.2 (Apple Git-55) From 0e9e821b4ef5119aed623019a173e71098a545e4 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Tue, 21 Apr 2015 14:27:10 -0700 Subject: [PATCH 05/24] mid-way through putting selector in SocketServer --- .../kafka/clients/ClusterConnectionStates.java | 78 +++++----- .../org/apache/kafka/clients/InFlightRequests.java | 18 +-- .../java/org/apache/kafka/clients/KafkaClient.java | 8 +- .../org/apache/kafka/clients/NetworkClient.java | 51 +++--- .../clients/consumer/internals/Coordinator.java | 4 +- .../kafka/clients/consumer/internals/Fetcher.java | 10 +- .../kafka/clients/producer/internals/Sender.java | 6 +- .../kafka/common/network/ByteBufferReceive.java | 6 +- .../kafka/common/network/ByteBufferSend.java | 6 +- .../kafka/common/network/NetworkReceive.java | 10 +- .../apache/kafka/common/network/NetworkSend.java | 4 +- .../org/apache/kafka/common/network/Receive.java | 2 +- .../apache/kafka/common/network/Selectable.java | 16 +- .../org/apache/kafka/common/network/Selector.java | 109 ++++++++----- .../java/org/apache/kafka/common/network/Send.java | 2 +- .../apache/kafka/common/requests/RequestSend.java | 2 +- .../apache/kafka/common/requests/ResponseSend.java | 2 +- .../java/org/apache/kafka/clients/MockClient.java | 6 +- .../apache/kafka/clients/NetworkClientTest.java | 8 +- .../apache/kafka/common/network/SelectorTest.java | 84 +++++----- .../java/org/apache/kafka/test/MockSelector.java | 25 +-- .../scala/kafka/api/ConsumerMetadataRequest.scala | 2 +- .../kafka/api/ControlledShutdownRequest.scala | 2 +- core/src/main/scala/kafka/api/FetchRequest.scala | 2 +- core/src/main/scala/kafka/api/FetchResponse.scala | 8 +- .../main/scala/kafka/api/LeaderAndIsrRequest.scala | 2 +- .../main/scala/kafka/api/OffsetCommitRequest.scala | 2 +- .../main/scala/kafka/api/OffsetFetchRequest.scala | 2 +- core/src/main/scala/kafka/api/OffsetRequest.scala | 2 +- .../src/main/scala/kafka/api/ProducerRequest.scala | 2 +- .../main/scala/kafka/api/StopReplicaRequest.scala | 2 +- .../scala/kafka/api/TopicMetadataRequest.scala | 2 +- .../scala/kafka/api/UpdateMetadataRequest.scala | 2 +- .../main/scala/kafka/network/BlockingChannel.scala | 8 +- .../kafka/network/RequestOrResponseSend.scala | 6 +- .../main/scala/kafka/network/SocketServer.scala | 171 ++------------------- .../main/scala/kafka/network/Transmission.scala | 2 +- .../main/scala/kafka/producer/SyncProducer.scala | 2 +- core/src/main/scala/kafka/server/KafkaApis.scala | 22 +-- .../main/scala/kafka/server/MessageSetSend.scala | 2 +- core/src/test/resources/log4j.properties | 6 +- 41 files changed, 306 insertions(+), 400 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/ClusterConnectionStates.java b/clients/src/main/java/org/apache/kafka/clients/ClusterConnectionStates.java index da76cc2..9ebda5e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/ClusterConnectionStates.java +++ b/clients/src/main/java/org/apache/kafka/clients/ClusterConnectionStates.java @@ -21,22 +21,22 @@ import java.util.Map; */ final class ClusterConnectionStates { private final long reconnectBackoffMs; - private final Map nodeState; + private final Map nodeState; public ClusterConnectionStates(long reconnectBackoffMs) { this.reconnectBackoffMs = reconnectBackoffMs; - this.nodeState = new HashMap(); + this.nodeState = new HashMap(); } /** - * Return true iff we can currently initiate a new connection to the given node. This will be the case if we are not + * Return true iff we can currently initiate a new connection. This will be the case if we are not * connected and haven't been connected for at least the minimum reconnection backoff period. - * @param node The node id to check + * @param id The connection id to check * @param now The current time in MS * @return true if we can initiate a new connection */ - public boolean canConnect(int node, long now) { - NodeConnectionState state = nodeState.get(node); + public boolean canConnect(String id, long now) { + NodeConnectionState state = nodeState.get(id); if (state == null) return true; else @@ -45,11 +45,11 @@ final class ClusterConnectionStates { /** * Return true if we are disconnected from the given node and can't re-establish a connection yet - * @param node The node to check + * @param id The connection to check * @param now The current time in ms */ - public boolean isBlackedOut(int node, long now) { - NodeConnectionState state = nodeState.get(node); + public boolean isBlackedOut(String id, long now) { + NodeConnectionState state = nodeState.get(id); if (state == null) return false; else @@ -60,11 +60,11 @@ final class ClusterConnectionStates { * Returns the number of milliseconds to wait, based on the connection state, before attempting to send data. When * disconnected, this respects the reconnect backoff time. When connecting or connected, this handles slow/stalled * connections. - * @param node The node to check + * @param id The connection to check * @param now The current time in ms */ - public long connectionDelay(int node, long now) { - NodeConnectionState state = nodeState.get(node); + public long connectionDelay(String id, long now) { + NodeConnectionState state = nodeState.get(id); if (state == null) return 0; long timeWaited = now - state.lastConnectAttemptMs; if (state.state == ConnectionState.DISCONNECTED) { @@ -77,67 +77,67 @@ final class ClusterConnectionStates { } /** - * Enter the connecting state for the given node. - * @param node The id of the node we are connecting to + * Enter the connecting state for the given connection. + * @param id The id of the connection * @param now The current time. */ - public void connecting(int node, long now) { - nodeState.put(node, new NodeConnectionState(ConnectionState.CONNECTING, now)); + public void connecting(String id, long now) { + nodeState.put(id, new NodeConnectionState(ConnectionState.CONNECTING, now)); } /** - * Return true iff we have a connection to the give node - * @param node The id of the node to check + * Return true iff a specific connection is connected + * @param id The id of the connection to check */ - public boolean isConnected(int node) { - NodeConnectionState state = nodeState.get(node); + public boolean isConnected(String id) { + NodeConnectionState state = nodeState.get(id); return state != null && state.state == ConnectionState.CONNECTED; } /** - * Return true iff we are in the process of connecting to the given node - * @param node The id of the node + * Return true iff we are in the process of connecting + * @param id The id of the connection */ - public boolean isConnecting(int node) { - NodeConnectionState state = nodeState.get(node); + public boolean isConnecting(String id) { + NodeConnectionState state = nodeState.get(id); return state != null && state.state == ConnectionState.CONNECTING; } /** - * Enter the connected state for the given node - * @param node The node we have connected to + * Enter the connected state for the given connection + * @param id The connection identifier */ - public void connected(int node) { - NodeConnectionState nodeState = nodeState(node); + public void connected(String id) { + NodeConnectionState nodeState = nodeState(id); nodeState.state = ConnectionState.CONNECTED; } /** * Enter the disconnected state for the given node - * @param node The node we have disconnected from + * @param id The connection we have disconnected */ - public void disconnected(int node) { - NodeConnectionState nodeState = nodeState(node); + public void disconnected(String id) { + NodeConnectionState nodeState = nodeState(id); nodeState.state = ConnectionState.DISCONNECTED; } /** - * Get the state of our connection to the given node - * @param node The id of the node + * Get the state of a given connection + * @param id The id of the connection * @return The state of our connection */ - public ConnectionState connectionState(int node) { - return nodeState(node).state; + public ConnectionState connectionState(String id) { + return nodeState(id).state; } /** * Get the state of a given node - * @param node The node to fetch the state for + * @param id The connection to fetch the state for */ - private NodeConnectionState nodeState(int node) { - NodeConnectionState state = this.nodeState.get(node); + private NodeConnectionState nodeState(String id) { + NodeConnectionState state = this.nodeState.get(id); if (state == null) - throw new IllegalStateException("No entry found for node " + node); + throw new IllegalStateException("No entry found for connection " + id); return state; } diff --git a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java index 936487b..15d00d4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java +++ b/clients/src/main/java/org/apache/kafka/clients/InFlightRequests.java @@ -24,14 +24,14 @@ import java.util.Map; final class InFlightRequests { private final int maxInFlightRequestsPerConnection; - private final Map> requests = new HashMap>(); + private final Map> requests = new HashMap>(); public InFlightRequests(int maxInFlightRequestsPerConnection) { this.maxInFlightRequestsPerConnection = maxInFlightRequestsPerConnection; } /** - * Add the given request to the queue for the node it was directed to + * Add the given request to the queue for the connection it was directed to */ public void add(ClientRequest request) { Deque reqs = this.requests.get(request.request().destination()); @@ -45,7 +45,7 @@ final class InFlightRequests { /** * Get the request queue for the given node */ - private Deque requestQueue(int node) { + private Deque requestQueue(String node) { Deque reqs = requests.get(node); if (reqs == null || reqs.isEmpty()) throw new IllegalStateException("Response from server for which there are no in-flight requests."); @@ -55,7 +55,7 @@ final class InFlightRequests { /** * Get the oldest request (the one that that will be completed next) for the given node */ - public ClientRequest completeNext(int node) { + public ClientRequest completeNext(String node) { return requestQueue(node).pollLast(); } @@ -63,7 +63,7 @@ final class InFlightRequests { * Get the last request we sent to the given node (but don't remove it from the queue) * @param node The node id */ - public ClientRequest lastSent(int node) { + public ClientRequest lastSent(String node) { return requestQueue(node).peekFirst(); } @@ -72,7 +72,7 @@ final class InFlightRequests { * @param node The node the request was sent to * @return The request */ - public ClientRequest completeLastSent(int node) { + public ClientRequest completeLastSent(String node) { return requestQueue(node).pollFirst(); } @@ -82,7 +82,7 @@ final class InFlightRequests { * @param node Node in question * @return true iff we have no requests still being sent to the given node */ - public boolean canSendMore(int node) { + public boolean canSendMore(String node) { Deque queue = requests.get(node); return queue == null || queue.isEmpty() || (queue.peekFirst().request().completed() && queue.size() < this.maxInFlightRequestsPerConnection); @@ -93,7 +93,7 @@ final class InFlightRequests { * @param node The node * @return The request count. */ - public int inFlightRequestCount(int node) { + public int inFlightRequestCount(String node) { Deque queue = requests.get(node); return queue == null ? 0 : queue.size(); } @@ -114,7 +114,7 @@ final class InFlightRequests { * @param node The node * @return All the in-flight requests for that node that have been removed */ - public Iterable clearAll(int node) { + public Iterable clearAll(String node) { Deque reqs = requests.get(node); if (reqs == null) { return Collections.emptyList(); diff --git a/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java b/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java index 96ac6d0..d98c84e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/KafkaClient.java @@ -80,13 +80,13 @@ public interface KafkaClient { public List poll(long timeout, long now); /** - * Complete all in-flight requests for a given node + * Complete all in-flight requests for a given connection * - * @param node The node to complete requests for + * @param id The connection to complete requests for * @param now The current time in ms * @return All requests that complete during this time period. */ - public List completeAll(int node, long now); + public List completeAll(String id, long now); /** * Complete all in-flight requests @@ -116,7 +116,7 @@ public interface KafkaClient { * * @param nodeId The id of the node */ - public int inFlightRequestCount(int nodeId); + public int inFlightRequestCount(String nodeId); /** * Generate a request header for the next request diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index b7ae595..84b9af5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ProtoUtils; import org.apache.kafka.common.protocol.types.Struct; @@ -111,7 +112,7 @@ public class NetworkClient implements KafkaClient { if (isReady(node, now)) return true; - if (connectionStates.canConnect(node.id(), now)) + if (connectionStates.canConnect(Integer.toString(node.id()), now)) // if we are interested in sending to a node and we don't have a connection to it, initiate one initiateConnect(node, now); @@ -129,7 +130,7 @@ public class NetworkClient implements KafkaClient { */ @Override public long connectionDelay(Node node, long now) { - return connectionStates.connectionDelay(node.id(), now); + return connectionStates.connectionDelay(Integer.toString(node.id()), now); } /** @@ -142,7 +143,7 @@ public class NetworkClient implements KafkaClient { */ @Override public boolean connectionFailed(Node node) { - return connectionStates.connectionState(node.id()).equals(ConnectionState.DISCONNECTED); + return connectionStates.connectionState(Integer.toString(node.id())).equals(ConnectionState.DISCONNECTED); } /** @@ -161,15 +162,15 @@ public class NetworkClient implements KafkaClient { return false; else // otherwise we are ready if we are connected and can send more requests - return isSendable(nodeId); + return isSendable(Integer.toString(nodeId)); } /** - * Are we connected and ready and able to send more requests to the given node? + * Are we connected and ready and able to send more requests to the given connection? * * @param node The node */ - private boolean isSendable(int node) { + private boolean isSendable(String node) { return connectionStates.isConnected(node) && inFlightRequests.canSendMore(node); } @@ -179,7 +180,7 @@ public class NetworkClient implements KafkaClient { * @param node The node to check * @return The connection state */ - public ConnectionState connectionState(int node) { + public ConnectionState connectionState(String node) { return connectionStates.connectionState(node); } @@ -190,7 +191,7 @@ public class NetworkClient implements KafkaClient { */ @Override public void send(ClientRequest request) { - int nodeId = request.request().destination(); + String nodeId = request.request().destination(); if (!isSendable(nodeId)) throw new IllegalStateException("Attempt to send a request to node " + nodeId + " which is not ready."); @@ -252,7 +253,7 @@ public class NetworkClient implements KafkaClient { * @return All the collected responses */ @Override - public List completeAll(int node, long now) { + public List completeAll(String node, long now) { try { this.selector.muteAll(); this.selector.unmute(node); @@ -288,7 +289,7 @@ public class NetworkClient implements KafkaClient { * Get the number of in-flight requests for a given node */ @Override - public int inFlightRequestCount(int nodeId) { + public int inFlightRequestCount(String nodeId) { return this.inFlightRequests.inFlightRequestCount(nodeId); } @@ -334,11 +335,11 @@ public class NetworkClient implements KafkaClient { for (int i = 0; i < nodes.size(); i++) { int idx = Utils.abs((this.nodeIndexOffset + i) % nodes.size()); Node node = nodes.get(idx); - int currInflight = this.inFlightRequests.inFlightRequestCount(node.id()); - if (currInflight == 0 && this.connectionStates.isConnected(node.id())) { + int currInflight = this.inFlightRequests.inFlightRequestCount(Integer.toString(node.id())); + if (currInflight == 0 && this.connectionStates.isConnected(Integer.toString(node.id()))) { // if we find an established connection with no in-flight requests we can stop right away return node; - } else if (!this.connectionStates.isBlackedOut(node.id(), now) && currInflight < inflight) { + } else if (!this.connectionStates.isBlackedOut(Integer.toString(node.id()), now) && currInflight < inflight) { // otherwise if this is the best we have found so far, record that inflight = currInflight; found = node; @@ -355,7 +356,7 @@ public class NetworkClient implements KafkaClient { */ private void handleCompletedSends(List responses, long now) { // if no response is expected then when the send is completed, return it - for (NetworkSend send : this.selector.completedSends()) { + for (Send send : this.selector.completedSends()) { ClientRequest request = this.inFlightRequests.lastSent(send.destination()); if (!request.expectResponse()) { this.inFlightRequests.completeLastSent(send.destination()); @@ -372,7 +373,7 @@ public class NetworkClient implements KafkaClient { */ private void handleCompletedReceives(List responses, long now) { for (NetworkReceive receive : this.selector.completedReceives()) { - int source = receive.source(); + String source = receive.source(); ClientRequest req = inFlightRequests.completeNext(source); ResponseHeader header = ResponseHeader.parse(receive.payload()); short apiKey = req.request().header().apiKey(); @@ -408,7 +409,7 @@ public class NetworkClient implements KafkaClient { * @param now The current time */ private void handleDisconnections(List responses, long now) { - for (int node : this.selector.disconnected()) { + for (String node : this.selector.disconnected()) { connectionStates.disconnected(node); log.debug("Node {} disconnected.", node); for (ClientRequest request : this.inFlightRequests.clearAll(node)) { @@ -429,7 +430,7 @@ public class NetworkClient implements KafkaClient { * Record any newly completed connections */ private void handleConnections() { - for (Integer id : this.selector.connected()) { + for (String id : this.selector.connected()) { log.debug("Completed connection to node {}", id); this.connectionStates.connected(id); } @@ -447,7 +448,7 @@ public class NetworkClient implements KafkaClient { /** * Create a metadata request for the given topics */ - private ClientRequest metadataRequest(long now, int node, Set topics) { + private ClientRequest metadataRequest(long now, String node, Set topics) { MetadataRequest metadata = new MetadataRequest(new ArrayList(topics)); RequestSend send = new RequestSend(node, nextRequestHeader(ApiKeys.METADATA), metadata.toStruct()); return new ClientRequest(now, true, send, null); @@ -460,6 +461,7 @@ public class NetworkClient implements KafkaClient { // Beware that the behavior of this method and the computation of timeouts for poll() are // highly dependent on the behavior of leastLoadedNode. Node node = this.leastLoadedNode(now); + String nodeConnectionId = Integer.toString(node.id()); if (node == null) { log.debug("Give up sending metadata request since no node is available"); // mark the timestamp for no node available to connect @@ -467,14 +469,14 @@ public class NetworkClient implements KafkaClient { return; } - if (connectionStates.isConnected(node.id()) && inFlightRequests.canSendMore(node.id())) { + if (connectionStates.isConnected(nodeConnectionId) && inFlightRequests.canSendMore(nodeConnectionId)) { Set topics = metadata.topics(); this.metadataFetchInProgress = true; - ClientRequest metadataRequest = metadataRequest(now, node.id(), topics); + ClientRequest metadataRequest = metadataRequest(now, nodeConnectionId, topics); log.debug("Sending metadata request {} to node {}", metadataRequest, node.id()); this.selector.send(metadataRequest.request()); this.inFlightRequests.add(metadataRequest); - } else if (connectionStates.canConnect(node.id(), now)) { + } else if (connectionStates.canConnect(nodeConnectionId, now)) { // we don't have a connection to this node right now, make one log.debug("Initialize connection to node {} for sending metadata request", node.id()); initiateConnect(node, now); @@ -493,16 +495,17 @@ public class NetworkClient implements KafkaClient { * Initiate a connection to the given node */ private void initiateConnect(Node node, long now) { + String nodeConnectionId = Integer.toString(node.id()); try { log.debug("Initiating connection to node {} at {}:{}.", node.id(), node.host(), node.port()); - this.connectionStates.connecting(node.id(), now); - selector.connect(node.id(), + this.connectionStates.connecting(nodeConnectionId, now); + selector.connect(Integer.toString(node.id()), new InetSocketAddress(node.host(), node.port()), this.socketSendBuffer, this.socketReceiveBuffer); } catch (IOException e) { /* attempt failed, we'll try again after the backoff */ - connectionStates.disconnected(node.id()); + connectionStates.disconnected(nodeConnectionId); /* maybe the problem is our metadata, update it */ metadata.requestUpdate(); log.debug("Error connecting to node {} at {}:{}:", node.id(), node.host(), node.port(), e); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java index e55ab11..637717b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java @@ -416,7 +416,7 @@ public final class Coordinator { log.debug("Issuing consumer metadata request to broker {}", node.id()); ConsumerMetadataRequest request = new ConsumerMetadataRequest(this.groupId); - RequestSend send = new RequestSend(node.id(), + RequestSend send = new RequestSend(Integer.toString(node.id()), this.client.nextRequestHeader(ApiKeys.CONSUMER_METADATA), request.toStruct()); long now = time.milliseconds(); @@ -435,7 +435,7 @@ public final class Coordinator { log.debug("Issuing request ({}: {}) to coordinator {}", api, request, this.consumerCoordinator.id()); RequestHeader header = this.client.nextRequestHeader(api); - RequestSend send = new RequestSend(this.consumerCoordinator.id(), header, request); + RequestSend send = new RequestSend(Integer.toString(this.consumerCoordinator.id()), header, request); return new ClientRequest(now, true, send, handler); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index ef9dd52..01188be 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -124,7 +124,7 @@ public class Fetcher { */ public void initFetches(Cluster cluster, long now) { for (ClientRequest request : createFetchRequests(cluster)) { - Node node = cluster.nodeById(request.request().destination()); + Node node = cluster.nodeById(Integer.parseInt(request.request().destination())); if (client.ready(node, now)) { log.trace("Initiating fetch to node {}: {}", node.id(), request); client.send(request); @@ -209,12 +209,12 @@ public class Fetcher { } else if (this.client.ready(info.leader(), now)) { Node node = info.leader(); ListOffsetRequest request = new ListOffsetRequest(-1, partitions); - RequestSend send = new RequestSend(node.id(), + RequestSend send = new RequestSend(Integer.toString(node.id()), this.client.nextRequestHeader(ApiKeys.LIST_OFFSETS), request.toStruct()); ClientRequest clientRequest = new ClientRequest(now, true, send, null); this.client.send(clientRequest); - List responses = this.client.completeAll(node.id(), now); + List responses = this.client.completeAll(Integer.toString(node.id()), now); if (responses.isEmpty()) throw new IllegalStateException("This should not happen."); ClientResponse response = responses.get(responses.size() - 1); @@ -257,7 +257,7 @@ public class Fetcher { for (TopicPartition partition : subscriptions.assignedPartitions()) { Node node = cluster.leaderFor(partition); // if there is a leader and no in-flight requests, issue a new fetch - if (node != null && this.client.inFlightRequestCount(node.id()) == 0) { + if (node != null && this.client.inFlightRequestCount(Integer.toString(node.id())) == 0) { Map fetch = fetchable.get(node.id()); if (fetch == null) { fetch = new HashMap(); @@ -273,7 +273,7 @@ public class Fetcher { for (Map.Entry> entry : fetchable.entrySet()) { int nodeId = entry.getKey(); final FetchRequest fetch = new FetchRequest(this.maxWaitMs, this.minBytes, entry.getValue()); - RequestSend send = new RequestSend(nodeId, this.client.nextRequestHeader(ApiKeys.FETCH), fetch.toStruct()); + RequestSend send = new RequestSend(Integer.toString(nodeId), this.client.nextRequestHeader(ApiKeys.FETCH), fetch.toStruct()); RequestCompletionHandler handler = new RequestCompletionHandler() { public void onComplete(ClientResponse response) { handleFetchResponse(response, fetch); diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 70954ca..fd25b07 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -302,7 +302,7 @@ public class Sender implements Runnable { recordsByPartition.put(tp, batch); } ProduceRequest request = new ProduceRequest(acks, timeout, produceRecordsByPartition); - RequestSend send = new RequestSend(destination, + RequestSend send = new RequestSend(Integer.toString(destination), this.client.nextRequestHeader(ApiKeys.PRODUCE), request.toStruct()); RequestCompletionHandler callback = new RequestCompletionHandler() { @@ -487,10 +487,10 @@ public class Sender implements Runnable { topicErrorSensor.record(count, now); } - public void recordLatency(int node, long latency) { + public void recordLatency(String node, long latency) { long now = time.milliseconds(); this.requestTimeSensor.record(latency, now); - if (node >= 0) { + if (!node.isEmpty()) { String nodeTimeName = "node-" + node + ".latency"; Sensor nodeRequestTime = this.metrics.getSensor(nodeTimeName); if (nodeRequestTime != null) diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java index 129ae82..cbb2f14 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java @@ -25,11 +25,11 @@ import java.nio.channels.ScatteringByteChannel; */ public class ByteBufferReceive implements Receive { - private final int source; + private final String source; private final ByteBuffer[] buffers; private int remaining; - public ByteBufferReceive(int source, ByteBuffer... buffers) { + public ByteBufferReceive(String source, ByteBuffer... buffers) { super(); this.source = source; this.buffers = buffers; @@ -38,7 +38,7 @@ public class ByteBufferReceive implements Receive { } @Override - public int source() { + public String source() { return source; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index e828833..45fadfc 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -22,12 +22,12 @@ import java.nio.channels.GatheringByteChannel; */ public class ByteBufferSend implements Send { - private final int destination; + private final String destination; protected final ByteBuffer[] buffers; private int remaining; private int size; - public ByteBufferSend(int destination, ByteBuffer... buffers) { + public ByteBufferSend(String destination, ByteBuffer... buffers) { super(); this.destination = destination; this.buffers = buffers; @@ -37,7 +37,7 @@ public class ByteBufferSend implements Send { } @Override - public int destination() { + public String destination() { return destination; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index cf97ea5..e891fb3 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -23,23 +23,23 @@ import java.nio.channels.ScatteringByteChannel; */ public class NetworkReceive implements Receive { - private final static int UNKNOWN_SOURCE = -1; + private final static String UNKNOWN_SOURCE = ""; private final static int UNLIMITED = -1; - private final int source; + private final String source; private final ByteBuffer size; private final long maxSize; private ByteBuffer buffer; - public NetworkReceive(int source, ByteBuffer buffer) { + public NetworkReceive(String source, ByteBuffer buffer) { this.source = source; this.buffer = buffer; this.size = null; this.maxSize = UNLIMITED; } - public NetworkReceive(int source) { + public NetworkReceive(String source) { this.source = source; this.size = ByteBuffer.allocate(4); this.buffer = null; @@ -58,7 +58,7 @@ public class NetworkReceive implements Receive { } @Override - public int source() { + public String source() { return source; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java index d1ee959..e5c4891 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java @@ -23,9 +23,9 @@ import java.nio.ByteBuffer; */ public class NetworkSend extends ByteBufferSend { - public final static int UNKNOWN_DEST = -1; + public final static String UNKNOWN_DEST = ""; - public NetworkSend(int destination, ByteBuffer... buffers) { + public NetworkSend(String destination, ByteBuffer... buffers) { super(destination, sizeDelimit(buffers)); } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Receive.java b/clients/src/main/java/org/apache/kafka/common/network/Receive.java index 4e33078..ac4e3f9 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Receive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Receive.java @@ -28,7 +28,7 @@ public interface Receive { /** * The numeric id of the source from which we are receiving data. */ - public int source(); + public String source(); /** * Are we done receiving data? diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java index b5f8d83..618a0fa 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java @@ -29,12 +29,12 @@ public interface Selectable { * @param receiveBufferSize The receive buffer for the socket * @throws IOException If we cannot begin connecting */ - public void connect(int id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException; + public void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException; /** * Begin disconnecting the connection identified by the given id */ - public void disconnect(int id); + public void disconnect(String id); /** * Wakeup this selector if it is blocked on I/O @@ -50,7 +50,7 @@ public interface Selectable { * Queue the given request for sending in the subsequent {@poll(long)} calls * @param send The request to send */ - public void send(NetworkSend send); + public void send(Send send); /** * Do I/O. Reads, writes, connection establishment, etc. @@ -62,7 +62,7 @@ public interface Selectable { /** * The list of sends that completed on the last {@link #poll(long, List) poll()} call. */ - public List completedSends(); + public List completedSends(); /** * The list of receives that completed on the last {@link #poll(long, List) poll()} call. @@ -73,25 +73,25 @@ public interface Selectable { * The list of connections that finished disconnecting on the last {@link #poll(long, List) poll()} * call. */ - public List disconnected(); + public List disconnected(); /** * The list of connections that completed their connection on the last {@link #poll(long, List) poll()} * call. */ - public List connected(); + public List connected(); /** * Disable reads from the given connection * @param id The id for the connection */ - public void mute(int id); + public void mute(String id); /** * Re-enable reads from the given connection * @param id The id for the connection */ - public void unmute(int id); + public void unmute(String id); /** * Disable reads from all connections diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 2051261..6ff7022 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -37,6 +37,7 @@ import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.metrics.stats.Count; import org.apache.kafka.common.metrics.stats.Max; import org.apache.kafka.common.metrics.stats.Rate; +import org.apache.kafka.common.utils.SystemTime; import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,16 +75,19 @@ public class Selector implements Selectable { private static final Logger log = LoggerFactory.getLogger(Selector.class); private final java.nio.channels.Selector nioSelector; - private final Map keys; - private final List completedSends; + private final Map keys; + private final List completedSends; private final List completedReceives; - private final List disconnected; - private final List connected; - private final List failedSends; + private final List disconnected; + private final List connected; + private final List failedSends; private final Time time; private final SelectorMetrics sensors; private final String metricGrpPrefix; private final Map metricTags; + private final Map lruConnections; + private long currentTimeNanos; + /** * Create a new nioSelector @@ -97,13 +101,15 @@ public class Selector implements Selectable { this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; - this.keys = new HashMap(); - this.completedSends = new ArrayList(); + this.keys = new HashMap(); + this.completedSends = new ArrayList(); this.completedReceives = new ArrayList(); - this.connected = new ArrayList(); - this.disconnected = new ArrayList(); - this.failedSends = new ArrayList(); + this.connected = new ArrayList(); + this.disconnected = new ArrayList(); + this.failedSends = new ArrayList(); this.sensors = new SelectorMetrics(metrics); + this.lruConnections = new LinkedHashMap(); + currentTimeNanos = new SystemTime().nanoseconds(); } /** @@ -120,7 +126,7 @@ public class Selector implements Selectable { * @throws IOException if DNS resolution fails on the hostname or if the broker is down */ @Override - public void connect(int id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException { + public void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException { if (this.keys.containsKey(id)) throw new IllegalStateException("There is already a connection for id " + id); @@ -148,11 +154,12 @@ public class Selector implements Selectable { /** * Register the nioSelector with an existing channel * Use this on server-side, when a connection is accepted by a different thread but processed by the Selector + * Note that we are not checking if the connection id is valid - since the connection already exists */ - public void register(int id, SocketChannel channel) throws ClosedChannelException { - if (log.isDebugEnabled()) - log.debug("Processor " + id + " listening to new connection from " + channel.socket().getRemoteSocketAddress()); - channel.register(nioSelector, SelectionKey.OP_READ); + public void register(String id, SocketChannel channel) throws ClosedChannelException { + SelectionKey key = channel.register(nioSelector, SelectionKey.OP_READ); + key.attach(new Transmissions(id)); + this.keys.put(id, key); } /** @@ -160,7 +167,7 @@ public class Selector implements Selectable { * processed until the next {@link #poll(long, List) poll()} call. */ @Override - public void disconnect(int id) { + public void disconnect(String id) { SelectionKey key = this.keys.get(id); if (key != null) key.cancel(); @@ -179,8 +186,8 @@ public class Selector implements Selectable { */ @Override public void close() { - for (SelectionKey key : this.nioSelector.keys()) - close(key); + for (String id: keys.keySet()) + close(id); try { this.nioSelector.close(); } catch (IOException e) { @@ -192,7 +199,7 @@ public class Selector implements Selectable { * Queue the given request for sending in the subsequent {@poll(long)} calls * @param send The request to send */ - public void send(NetworkSend send) { + public void send(Send send) { SelectionKey key = keyForId(send.destination()); Transmissions transmissions = transmissions(key); if (transmissions.hasSend()) @@ -201,7 +208,7 @@ public class Selector implements Selectable { try { key.interestOps(key.interestOps() | SelectionKey.OP_WRITE); } catch (CancelledKeyException e) { - close(key); + close(transmissions.id); this.failedSends.add(send.destination()); } } @@ -227,6 +234,7 @@ public class Selector implements Selectable { long startSelect = time.nanoseconds(); int readyKeys = select(timeout); long endSelect = time.nanoseconds(); + currentTimeNanos = endSelect; this.sensors.selectTime.record(endSelect - startSelect, time.milliseconds()); if (readyKeys > 0) { @@ -241,7 +249,7 @@ public class Selector implements Selectable { // register all per-broker metrics at once sensors.maybeRegisterNodeMetrics(transmissions.id); - + lruConnections.put(transmissions.id, currentTimeNanos); try { /* complete any connections that have finished their handshake */ if (key.isConnectable()) { @@ -269,7 +277,8 @@ public class Selector implements Selectable { transmissions.send.writeTo(channel); if (transmissions.send.remaining() <= 0) { this.completedSends.add(transmissions.send); - this.sensors.recordBytesSent(transmissions.id, transmissions.send.size()); + // TODO: Figure out the metrics here. Send doesn't have "size" in general and the server-side records sent messages, not sent bytes. + this.sensors.recordBytesSent(transmissions.id, 0); transmissions.clearSend(); key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE); } @@ -277,7 +286,7 @@ public class Selector implements Selectable { /* cancel any defunct sockets */ if (!key.isValid()) { - close(key); + close(transmissions.id); this.disconnected.add(transmissions.id); } } catch (IOException e) { @@ -286,7 +295,7 @@ public class Selector implements Selectable { log.info("Connection {} disconnected", desc); else log.warn("Error in I/O with connection to {}", desc, e); - close(key); + close(transmissions.id); this.disconnected.add(transmissions.id); } } @@ -306,7 +315,7 @@ public class Selector implements Selectable { } @Override - public List completedSends() { + public List completedSends() { return this.completedSends; } @@ -316,17 +325,17 @@ public class Selector implements Selectable { } @Override - public List disconnected() { + public List disconnected() { return this.disconnected; } @Override - public List connected() { + public List connected() { return this.connected; } @Override - public void mute(int id) { + public void mute(String id) { mute(this.keyForId(id)); } @@ -335,7 +344,7 @@ public class Selector implements Selectable { } @Override - public void unmute(int id) { + public void unmute(String id) { unmute(this.keyForId(id)); } @@ -355,6 +364,24 @@ public class Selector implements Selectable { unmute(key); } + public long maybeCloseOldestConnection(long connectionsMaxIdleNanos) { + if(lruConnections.isEmpty()) { + return currentTimeNanos + connectionsMaxIdleNanos; + } else { + Map.Entry oldestConnectionEntry = lruConnections.entrySet().iterator().next(); + Long connectionLastActiveTime = oldestConnectionEntry.getValue(); + Long nextIdleCloseCheckTime = connectionLastActiveTime + connectionsMaxIdleNanos; + if(currentTimeNanos > nextIdleCloseCheckTime) { + String connectionId = oldestConnectionEntry.getKey(); + if (log.isTraceEnabled()) + log.trace("About to close the idle connection from " + connectionId + + " due to being idle for " + (currentTimeNanos - connectionLastActiveTime) / 1000 / 1000 + " millis"); + close(connectionId); + } + return nextIdleCloseCheckTime; + } + } + /** * Clear the results from the prior poll */ @@ -386,7 +413,9 @@ public class Selector implements Selectable { /** * Begin closing this connection */ - private void close(SelectionKey key) { + private void close(String id) { + SelectionKey key = keyForId(id); + lruConnections.remove(id); SocketChannel channel = channel(key); Transmissions trans = transmissions(key); if (trans != null) { @@ -408,7 +437,7 @@ public class Selector implements Selectable { /** * Get the selection key associated with this numeric id */ - private SelectionKey keyForId(int id) { + private SelectionKey keyForId(String id) { SelectionKey key = this.keys.get(id); if (key == null) throw new IllegalStateException("Attempt to write to socket for which there is no open connection."); @@ -433,11 +462,11 @@ public class Selector implements Selectable { * The id and in-progress send and receive associated with a connection */ private static class Transmissions { - public int id; - public NetworkSend send; + public String id; + public Send send; public NetworkReceive receive; - public Transmissions(int id) { + public Transmissions(String id) { this.id = id; } @@ -522,8 +551,8 @@ public class Selector implements Selectable { }); } - public void maybeRegisterNodeMetrics(int node) { - if (node >= 0) { + public void maybeRegisterNodeMetrics(String node) { + if (!node.isEmpty()) { // if one sensor of the metrics has been registered for the node, // then all other sensors should have been registered; and vice versa String nodeRequestName = "node-" + node + ".bytes-sent"; @@ -561,10 +590,10 @@ public class Selector implements Selectable { } } - public void recordBytesSent(int node, int bytes) { + public void recordBytesSent(String node, int bytes) { long now = time.milliseconds(); this.bytesSent.record(bytes, now); - if (node >= 0) { + if (!node.isEmpty()) { String nodeRequestName = "node-" + node + ".bytes-sent"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest != null) @@ -572,10 +601,10 @@ public class Selector implements Selectable { } } - public void recordBytesReceived(int node, int bytes) { + public void recordBytesReceived(String node, int bytes) { long now = time.milliseconds(); this.bytesReceived.record(bytes, now); - if (node >= 0) { + if (!node.isEmpty()) { String nodeRequestName = "node-" + node + ".bytes-received"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest != null) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index 234945b..a94d9c3 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -24,7 +24,7 @@ public interface Send { /** * The numeric id for the destination of this send */ - public int destination(); + public String destination(); /** * The number of bytes remaining to send diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java index 27cbf39..3fec60b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestSend.java @@ -25,7 +25,7 @@ public class RequestSend extends NetworkSend { private final RequestHeader header; private final Struct body; - public RequestSend(int destination, RequestHeader header, Struct body) { + public RequestSend(String destination, RequestHeader header, Struct body) { super(destination, serialize(header, body)); this.header = header; this.body = body; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java index c510559..e242f93 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseSend.java @@ -23,7 +23,7 @@ import java.nio.ByteBuffer; public class ResponseSend extends NetworkSend { - public ResponseSend(int destination, ResponseHeader header, Struct body) { + public ResponseSend(String destination, ResponseHeader header, Struct body) { super(destination, serialize(header, body)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java index 5e3fab1..d9c97e9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java @@ -78,7 +78,7 @@ public class MockClient implements KafkaClient { return false; } - public void disconnect(Integer node) { + public void disconnect(String node) { Iterator iter = requests.iterator(); while (iter.hasNext()) { ClientRequest request = iter.next(); @@ -115,7 +115,7 @@ public class MockClient implements KafkaClient { } @Override - public List completeAll(int node, long now) { + public List completeAll(String node, long now) { return completeAll(now); } @@ -158,7 +158,7 @@ public class MockClient implements KafkaClient { } @Override - public int inFlightRequestCount(int nodeId) { + public int inFlightRequestCount(String nodeId) { return requests.size(); } diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index 8b27889..cfc1303 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -65,7 +65,7 @@ public class NetworkClientTest { client.poll(1, time.milliseconds()); selector.clear(); assertTrue("Now the client is ready", client.ready(node, time.milliseconds())); - selector.disconnect(node.id()); + selector.disconnect(Integer.toString(node.id())); client.poll(1, time.milliseconds()); selector.clear(); assertFalse("After we forced the disconnection the client is no longer ready.", client.ready(node, time.milliseconds())); @@ -74,7 +74,7 @@ public class NetworkClientTest { @Test(expected = IllegalStateException.class) public void testSendToUnreadyNode() { - RequestSend send = new RequestSend(5, + RequestSend send = new RequestSend("5", client.nextRequestHeader(ApiKeys.METADATA), new MetadataRequest(Arrays.asList("test")).toStruct()); ClientRequest request = new ClientRequest(time.milliseconds(), false, send, null); @@ -86,7 +86,7 @@ public class NetworkClientTest { public void testSimpleRequestResponse() { ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.emptyMap()); RequestHeader reqHeader = client.nextRequestHeader(ApiKeys.PRODUCE); - RequestSend send = new RequestSend(node.id(), reqHeader, produceRequest.toStruct()); + RequestSend send = new RequestSend(Integer.toString(node.id()), reqHeader, produceRequest.toStruct()); TestCallbackHandler handler = new TestCallbackHandler(); ClientRequest request = new ClientRequest(time.milliseconds(), true, send, handler); awaitReady(client, node); @@ -101,7 +101,7 @@ public class NetworkClientTest { respHeader.writeTo(buffer); resp.writeTo(buffer); buffer.flip(); - selector.completeReceive(new NetworkReceive(node.id(), buffer)); + selector.completeReceive(new NetworkReceive(Integer.toString(node.id()), buffer)); List responses = client.poll(1, time.milliseconds()); assertEquals(1, responses.size()); assertTrue("The handler should have executed.", handler.executed); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index d5b306b..5a49975 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -22,10 +22,7 @@ import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.*; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.utils.MockTime; @@ -63,7 +60,7 @@ public class SelectorTest { */ @Test public void testServerDisconnect() throws Exception { - int node = 0; + String node = "0"; // connect and do a simple request blockingConnect(node); @@ -84,7 +81,7 @@ public class SelectorTest { */ @Test public void testClientDisconnect() throws Exception { - int node = 0; + String node = "0"; blockingConnect(node); selector.disconnect(node); selector.send(createSend(node, "hello1")); @@ -101,7 +98,7 @@ public class SelectorTest { */ @Test(expected = IllegalStateException.class) public void testCantSendWithInProgress() throws Exception { - int node = 0; + String node = "0"; blockingConnect(node); selector.send(createSend(node, "test1")); selector.send(createSend(node, "test2")); @@ -113,7 +110,7 @@ public class SelectorTest { */ @Test(expected = IllegalStateException.class) public void testCantSendWithoutConnecting() throws Exception { - selector.send(createSend(0, "test")); + selector.send(createSend("0", "test")); selector.poll(1000L); } @@ -122,7 +119,7 @@ public class SelectorTest { */ @Test(expected = IOException.class) public void testNoRouteToHost() throws Exception { - selector.connect(0, new InetSocketAddress("asdf.asdf.dsc", server.port), BUFFER_SIZE, BUFFER_SIZE); + selector.connect("0", new InetSocketAddress("asdf.asdf.dsc", server.port), BUFFER_SIZE, BUFFER_SIZE); } /** @@ -130,7 +127,7 @@ public class SelectorTest { */ @Test public void testConnectionRefused() throws Exception { - int node = 0; + String node = "0"; ServerSocket nonListeningSocket = new ServerSocket(0); int nonListeningPort = nonListeningSocket.getLocalPort(); selector.connect(node, new InetSocketAddress("localhost", nonListeningPort), BUFFER_SIZE, BUFFER_SIZE); @@ -151,14 +148,15 @@ public class SelectorTest { // create connections InetSocketAddress addr = new InetSocketAddress("localhost", server.port); for (int i = 0; i < conns; i++) - selector.connect(i, addr, BUFFER_SIZE, BUFFER_SIZE); - + selector.connect(Integer.toString(i), addr, BUFFER_SIZE, BUFFER_SIZE); // send echo requests and receive responses - int[] requests = new int[conns]; - int[] responses = new int[conns]; + Map requests = new HashMap(); + Map responses = new HashMap(); int responseCount = 0; - for (int i = 0; i < conns; i++) - selector.send(createSend(i, i + "-" + 0)); + for (int i = 0; i < conns; i++) { + String node = Integer.toString(i); + selector.send(createSend(node, node + "-0")); + } // loop until we complete all requests while (responseCount < conns * reqs) { @@ -171,19 +169,27 @@ public class SelectorTest { for (NetworkReceive receive : selector.completedReceives()) { String[] pieces = asString(receive).split("-"); assertEquals("Should be in the form 'conn-counter'", 2, pieces.length); - assertEquals("Check the source", receive.source(), Integer.parseInt(pieces[0])); + assertEquals("Check the source", receive.source(), pieces[0]); assertEquals("Check that the receive has kindly been rewound", 0, receive.payload().position()); - assertEquals("Check the request counter", responses[receive.source()], Integer.parseInt(pieces[1])); - responses[receive.source()]++; // increment the expected counter + if (responses.containsKey(receive.source())) { + assertEquals("Check the request counter", (int) responses.get(receive.source()), Integer.parseInt(pieces[1])); + responses.put(receive.source(), responses.get(receive.source()) + 1); + } else { + assertEquals("Check the request counter", 0, Integer.parseInt(pieces[1])); + responses.put(receive.source(), 1); + } responseCount++; } // prepare new sends for the next round - for (NetworkSend send : selector.completedSends()) { - int dest = send.destination(); - requests[dest]++; - if (requests[dest] < reqs) - selector.send(createSend(dest, dest + "-" + requests[dest])); + for (Send send : selector.completedSends()) { + String dest = send.destination(); + if (requests.containsKey(dest)) + requests.put(dest, requests.get(dest) + 1); + else + requests.put(dest, 1); + if (requests.get(dest) < reqs) + selector.send(createSend(dest, dest + "-" + requests.get(dest))); } } } @@ -193,7 +199,7 @@ public class SelectorTest { */ @Test public void testSendLargeRequest() throws Exception { - int node = 0; + String node = "0"; blockingConnect(node); String big = TestUtils.randomString(10 * BUFFER_SIZE); assertEquals(big, blockingRequest(node, big)); @@ -204,41 +210,41 @@ public class SelectorTest { */ @Test public void testEmptyRequest() throws Exception { - int node = 0; + String node = "0"; blockingConnect(node); assertEquals("", blockingRequest(node, "")); } @Test(expected = IllegalStateException.class) public void testExistingConnectionId() throws IOException { - blockingConnect(0); - blockingConnect(0); + blockingConnect("0"); + blockingConnect("0"); } @Test public void testMute() throws Exception { - blockingConnect(0); - blockingConnect(1); + blockingConnect("0"); + blockingConnect("1"); - selector.send(createSend(0, "hello")); - selector.send(createSend(1, "hi")); + selector.send(createSend("0", "hello")); + selector.send(createSend("1", "hi")); - selector.mute(1); + selector.mute("1"); while (selector.completedReceives().isEmpty()) selector.poll(5); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should not be from the muted node", 0, selector.completedReceives().get(0).source()); + assertEquals("The response should not be from the muted node", "0", selector.completedReceives().get(0).source()); - selector.unmute(1); + selector.unmute("1"); do { selector.poll(5); } while (selector.completedReceives().isEmpty()); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should be from the previously muted node", 1, selector.completedReceives().get(0).source()); + assertEquals("The response should be from the previously muted node", "1", selector.completedReceives().get(0).source()); } - private String blockingRequest(int node, String s) throws IOException { + private String blockingRequest(String node, String s) throws IOException { selector.send(createSend(node, s)); selector.poll(1000L); while (true) { @@ -250,13 +256,13 @@ public class SelectorTest { } /* connect and wait for the connection to complete */ - private void blockingConnect(int node) throws IOException { + private void blockingConnect(String node) throws IOException { selector.connect(node, new InetSocketAddress("localhost", server.port), BUFFER_SIZE, BUFFER_SIZE); while (!selector.connected().contains(node)) selector.poll(10000L); } - private NetworkSend createSend(int node, String s) { + private NetworkSend createSend(String node, String s) { return new NetworkSend(node, ByteBuffer.wrap(s.getBytes())); } diff --git a/clients/src/test/java/org/apache/kafka/test/MockSelector.java b/clients/src/test/java/org/apache/kafka/test/MockSelector.java index ea89b06..51eb9d1 100644 --- a/clients/src/test/java/org/apache/kafka/test/MockSelector.java +++ b/clients/src/test/java/org/apache/kafka/test/MockSelector.java @@ -20,6 +20,7 @@ import java.util.List; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Selectable; +import org.apache.kafka.common.network.Send; import org.apache.kafka.common.utils.Time; /** @@ -28,23 +29,23 @@ import org.apache.kafka.common.utils.Time; public class MockSelector implements Selectable { private final Time time; - private final List initiatedSends = new ArrayList(); - private final List completedSends = new ArrayList(); + private final List initiatedSends = new ArrayList(); + private final List completedSends = new ArrayList(); private final List completedReceives = new ArrayList(); - private final List disconnected = new ArrayList(); - private final List connected = new ArrayList(); + private final List disconnected = new ArrayList(); + private final List connected = new ArrayList(); public MockSelector(Time time) { this.time = time; } @Override - public void connect(int id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException { + public void connect(String id, InetSocketAddress address, int sendBufferSize, int receiveBufferSize) throws IOException { this.connected.add(id); } @Override - public void disconnect(int id) { + public void disconnect(String id) { this.disconnected.add(id); } @@ -64,7 +65,7 @@ public class MockSelector implements Selectable { } @Override - public void send(NetworkSend send) { + public void send(Send send) { this.initiatedSends.add(send); } @@ -76,7 +77,7 @@ public class MockSelector implements Selectable { } @Override - public List completedSends() { + public List completedSends() { return completedSends; } @@ -94,21 +95,21 @@ public class MockSelector implements Selectable { } @Override - public List disconnected() { + public List disconnected() { return disconnected; } @Override - public List connected() { + public List connected() { return connected; } @Override - public void mute(int id) { + public void mute(String id) { } @Override - public void unmute(int id) { + public void unmute(String id) { } @Override diff --git a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala index 05cda52..e221870 100644 --- a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala @@ -65,7 +65,7 @@ case class ConsumerMetadataRequest(group: String, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { // return ConsumerCoordinatorNotAvailable for all uncaught errors val errorResponse = ConsumerMetadataResponse(None, ErrorMapping.ConsumerCoordinatorNotAvailableCode, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } def describe(details: Boolean) = { diff --git a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala index ff4ddb9..e031b63 100644 --- a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala +++ b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala @@ -62,7 +62,7 @@ case class ControlledShutdownRequest(val versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = ControlledShutdownResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]]), Set.empty[TopicAndPartition]) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean = false): String = { diff --git a/core/src/main/scala/kafka/api/FetchRequest.scala b/core/src/main/scala/kafka/api/FetchRequest.scala index b038c15..033865e 100644 --- a/core/src/main/scala/kafka/api/FetchRequest.scala +++ b/core/src/main/scala/kafka/api/FetchRequest.scala @@ -149,7 +149,7 @@ case class FetchRequest(versionId: Short = FetchRequest.CurrentVersion, (topicAndPartition, FetchResponsePartitionData(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1, MessageSet.Empty)) } val errorResponse = FetchResponse(correlationId, fetchResponsePartitionData) - requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(errorResponse))) + requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 002b17c..203ffc3 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -65,7 +65,7 @@ class PartitionDataSend(val partitionId: Int, override def completed = !buffer.hasRemaining && messagesSentSize >= messageSize - override def destination: Int = -1 + override def destination: String = "" override def reify = null @@ -115,7 +115,7 @@ class TopicDataSend(val topicData: TopicData) extends Send { override def completed: Boolean = sent >= size - override def destination: Int = -1 + override def destination: String = "" override def reify = null @@ -213,7 +213,7 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR } -class FetchResponseSend(val fetchResponse: FetchResponse) extends Send { +class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { private val size = fetchResponse.sizeInBytes private var sent = 0L @@ -222,7 +222,7 @@ class FetchResponseSend(val fetchResponse: FetchResponse) extends Send { override def completed = sent >= sendSize - override def destination = -1 + override def destination = dest override def reify = null diff --git a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala index e03bdb0..ec4b992 100644 --- a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala +++ b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala @@ -186,7 +186,7 @@ case class LeaderAndIsrRequest (versionId: Short, case (topicAndPartition, partitionAndState) => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = LeaderAndIsrResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala index 226b8f8..5e4188e 100644 --- a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala @@ -163,7 +163,7 @@ case class OffsetCommitRequest(groupId: String, val commitStatus = requestInfo.mapValues(_ => errorCode) val commitResponse = OffsetCommitResponse(commitStatus, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(commitResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, commitResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala index f179903..6186374 100644 --- a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala @@ -96,7 +96,7 @@ case class OffsetFetchRequest(groupId: String, )) }.toMap val errorResponse = OffsetFetchResponse(requestInfo=responseMap, correlationId=correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetRequest.scala b/core/src/main/scala/kafka/api/OffsetRequest.scala index b1bf46a..45c5c7d 100644 --- a/core/src/main/scala/kafka/api/OffsetRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetRequest.scala @@ -118,7 +118,7 @@ case class OffsetRequest(requestInfo: Map[TopicAndPartition, PartitionOffsetRequ (topicAndPartition, PartitionOffsetsResponse(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), null)) } val errorResponse = OffsetResponse(correlationId, partitionOffsetResponseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/ProducerRequest.scala b/core/src/main/scala/kafka/api/ProducerRequest.scala index f5167e4..41ff006 100644 --- a/core/src/main/scala/kafka/api/ProducerRequest.scala +++ b/core/src/main/scala/kafka/api/ProducerRequest.scala @@ -137,7 +137,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, (topicAndPartition, ProducerResponseStatus(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1l)) } val errorResponse = ProducerResponse(correlationId, producerResponseStatus) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } } diff --git a/core/src/main/scala/kafka/api/StopReplicaRequest.scala b/core/src/main/scala/kafka/api/StopReplicaRequest.scala index aee6973..ec914a9 100644 --- a/core/src/main/scala/kafka/api/StopReplicaRequest.scala +++ b/core/src/main/scala/kafka/api/StopReplicaRequest.scala @@ -106,7 +106,7 @@ case class StopReplicaRequest(versionId: Short, case topicAndPartition => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) }.toMap val errorResponse = StopReplicaResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala index 3cdbd40..fc68b7b 100644 --- a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala @@ -82,7 +82,7 @@ case class TopicMetadataRequest(val versionId: Short, topic => TopicMetadata(topic, Nil, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = TopicMetadataResponse(Seq(), topicMetadata, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala index e463395..42cabcd 100644 --- a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala @@ -128,7 +128,7 @@ case class UpdateMetadataRequest (versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = new UpdateMetadataResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]])) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/network/BlockingChannel.scala b/core/src/main/scala/kafka/network/BlockingChannel.scala index 45adb92..8139d3a 100644 --- a/core/src/main/scala/kafka/network/BlockingChannel.scala +++ b/core/src/main/scala/kafka/network/BlockingChannel.scala @@ -44,6 +44,7 @@ class BlockingChannel( val host: String, private var writeChannel: GatheringByteChannel = null private val lock = new Object() private val connectTimeoutMs = readTimeoutMs + private var connectionId: String = "" def connect() = lock synchronized { if(!connected) { @@ -64,6 +65,11 @@ class BlockingChannel( val host: String, // See: http://stackoverflow.com/questions/2866557/timeout-for-socketchannel-doesnt-work readChannel = Channels.newChannel(channel.socket().getInputStream) connected = true + val localHost = channel.socket.getLocalAddress.getHostAddress + val localPort = channel.socket.getLocalPort + val remoteHost = channel.socket.getInetAddress.getHostAddress + val remotePort = channel.socket.getPort + connectionId = localHost + ":" + localPort + "-" + remoteHost + ":" + remotePort // settings may not match what we requested above val msg = "Created socket with SO_TIMEOUT = %d (requested %d), SO_RCVBUF = %d (requested %d), SO_SNDBUF = %d (requested %d), connectTimeoutMs = %d." debug(msg.format(channel.socket.getSoTimeout, @@ -102,7 +108,7 @@ class BlockingChannel( val host: String, if(!connected) throw new ClosedChannelException() - val send = new RequestOrResponseSend(request) + val send = new RequestOrResponseSend(connectionId, request) send.writeCompletely(writeChannel) } diff --git a/core/src/main/scala/kafka/network/RequestOrResponseSend.scala b/core/src/main/scala/kafka/network/RequestOrResponseSend.scala index 424258f..364f24b 100644 --- a/core/src/main/scala/kafka/network/RequestOrResponseSend.scala +++ b/core/src/main/scala/kafka/network/RequestOrResponseSend.scala @@ -38,10 +38,10 @@ object RequestOrResponseSend { } } -class RequestOrResponseSend(val buffer: ByteBuffer) extends NetworkSend(buffer) with Logging { +class RequestOrResponseSend(val dest: String, val buffer: ByteBuffer) extends NetworkSend(dest, buffer) with Logging { - def this(request: RequestOrResponse) { - this(RequestOrResponseSend.serialize(request)) + def this(dest: String, request: RequestOrResponse) { + this(dest, RequestOrResponseSend.serialize(request)) } def writeCompletely(channel: GatheringByteChannel): Long = { diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 4f1b232..9cf2068 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -349,94 +349,24 @@ private[kafka] class Processor(val id: Int, // register any new responses for writing processNewResponses() val startSelectTime = new SystemTime().nanoseconds - val ready = selector.select(300) - currentTimeNanos = new SystemTime().nanoseconds - val idleTime = currentTimeNanos - startSelectTime - idleMeter.mark(idleTime) - // We use a single meter for aggregate idle percentage for the thread pool. - // Since meter is calculated as total_recorded_value / time_window and - // time_window is independent of the number of threads, each recorded idle - // time should be discounted by # threads. - aggregateIdleMeter.mark(idleTime / totalProcessorThreads) - - trace("Processor id " + id + " selection time = " + idleTime + " ns") - if(ready > 0) { - val keys = selector.selectedKeys() - val iter = keys.iterator() - while(iter.hasNext && isRunning) { - var key: SelectionKey = null - try { - key = iter.next - iter.remove() - if(key.isReadable) - read(key) - else if(key.isWritable) - write(key) - else if(!key.isValid) - close(key) - else - throw new IllegalStateException("Unrecognized key state for processor thread.") - } catch { - case e: EOFException => { - info("Closing socket connection to %s.".format(channelFor(key).socket.getInetAddress)) - close(key) - } case e: InvalidRequestException => { - info("Closing socket connection to %s due to invalid request: %s".format(channelFor(key).socket.getInetAddress, e.getMessage)) - close(key) - } case e: Throwable => { - error("Closing socket for " + channelFor(key).socket.getInetAddress + " because of error", e) - close(key) - } - } - } - } + val ready = selector.poll(300) + collection.JavaConversions.iterableAsScalaIterable(selector.completedReceives()).foreach( receive => { + val req = RequestChannel.Request(processor = id, requestKey = receive.source(), buffer = receive.payload(), startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) + requestChannel.sendRequest(req) + }) + maybeCloseOldestConnection } - debug("Closing selector.") + debug("Closing selector - processor " + id) closeAll() - swallowError(selector.close()) shutdownComplete() } - /** - * Close the given key and associated socket - */ - override def close(key: SelectionKey): Unit = { - lruConnections.remove(key) - super.close(key) - } - private def processNewResponses() { var curr = requestChannel.receiveResponse(id) while(curr != null) { - val key = curr.request.requestKey.asInstanceOf[SelectionKey] try { - curr.responseAction match { - case RequestChannel.NoOpAction => { - // There is no response to send to the client, we need to read more pipelined requests - // that are sitting in the server's socket buffer - curr.request.updateRequestMetrics - trace("Socket server received empty response to send, registering for read: " + curr) - key.interestOps(SelectionKey.OP_READ) - key.attach(null) - } - case RequestChannel.SendAction => { - trace("Socket server received response to send, registering for write: " + curr) - key.interestOps(SelectionKey.OP_WRITE) - key.attach(curr) - } - case RequestChannel.CloseConnectionAction => { - curr.request.updateRequestMetrics - trace("Closing socket connection actively according to the response code.") - close(key) - } - case responseCode => throw new KafkaException("No mapping found for response code " + responseCode) - } - } catch { - case e: CancelledKeyException => { - debug("Ignoring response for closed socket.") - close(key) - } + selector.send(curr.responseSend) } finally { curr = requestChannel.receiveResponse(id) } @@ -458,63 +388,12 @@ private[kafka] class Processor(val id: Int, while(!newConnections.isEmpty) { val channel = newConnections.poll() debug("Processor " + id + " listening to new connection from " + channel.socket.getRemoteSocketAddress) - channel.register(selector, SelectionKey.OP_READ) - } - } - - /* - * Process reads from ready sockets - */ - def read(key: SelectionKey) { - lruConnections.put(key, currentTimeNanos) - val socketChannel = channelFor(key) - var receive = key.attachment.asInstanceOf[NetworkReceive] - if(key.attachment == null) { - receive = new NetworkReceive(maxRequestSize.toLong) - key.attach(receive) - } - val read = receive.readFrom(socketChannel) - val address = socketChannel.socket.getRemoteSocketAddress(); - trace(read + " bytes read from " + address) - if(read < 0) { - close(key) - } else if(receive.complete) { - val port = socketChannel.socket().getLocalPort - val protocol = portToProtocol.get(port) - receive.payload().rewind() - val req = RequestChannel.Request(processor = id, requestKey = key, buffer = receive.payload(), startTimeMs = time.milliseconds, remoteAddress = address, securityProtocol = protocol) - requestChannel.sendRequest(req) - key.attach(null) - // explicitly reset interest ops to not READ, no need to wake up the selector just yet - key.interestOps(key.interestOps & (~SelectionKey.OP_READ)) - } else { - // more reading to be done - trace("Did not finish reading, registering for read again on connection " + socketChannel.socket.getRemoteSocketAddress()) - key.interestOps(SelectionKey.OP_READ) - wakeup() - } - } - - /* - * Process writes to ready sockets - */ - def write(key: SelectionKey) { - val socketChannel = channelFor(key) - val response = key.attachment().asInstanceOf[RequestChannel.Response] - val responseSend = response.responseSend - if(responseSend == null) - throw new IllegalStateException("Registered for write interest but no response attached to key.") - val written = responseSend.writeTo(socketChannel) - trace(written + " bytes written to " + socketChannel.socket.getRemoteSocketAddress() + " using key " + key) - if(responseSend.completed()) { - response.request.updateRequestMetrics() - key.attach(null) - trace("Finished writing, registering for read on connection " + socketChannel.socket.getRemoteSocketAddress()) - key.interestOps(SelectionKey.OP_READ) - } else { - trace("Did not finish writing, registering for write again on connection " + socketChannel.socket.getRemoteSocketAddress()) - key.interestOps(SelectionKey.OP_WRITE) - wakeup() + val localHost = channel.socket().getLocalAddress.getHostAddress + val localPort = channel.socket().getLocalPort + val remoteHost = channel.socket().getInetAddress.getHostAddress + val remotePort = channel.socket().getPort + val connectionId = localHost + ":" + localPort + "-" + remoteHost + ":" + remotePort + selector.register(connectionId, channel) } } @@ -522,19 +401,7 @@ private[kafka] class Processor(val id: Int, private def maybeCloseOldestConnection { if(currentTimeNanos > nextIdleCloseCheckTime) { - if(lruConnections.isEmpty) { - nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos - } else { - val oldestConnectionEntry = lruConnections.entrySet.iterator().next() - val connectionLastActiveTime = oldestConnectionEntry.getValue - nextIdleCloseCheckTime = connectionLastActiveTime + connectionsMaxIdleNanos - if(currentTimeNanos > nextIdleCloseCheckTime) { - val key: SelectionKey = oldestConnectionEntry.getKey - trace("About to close the idle connection from " + key.channel.asInstanceOf[SocketChannel].socket.getRemoteSocketAddress - + " due to being idle for " + (currentTimeNanos - connectionLastActiveTime) / 1000 / 1000 + " millis") - close(key) - } - } + nextIdleCloseCheckTime = selector.maybeCloseOldestConnection(connectionsMaxIdleNanos) } } @@ -542,13 +409,7 @@ private[kafka] class Processor(val id: Int, * Close all open connections */ def closeAll() { - // removes cancelled keys from selector.keys set - this.selector.selectNow() - val iter = this.selector.keys().iterator() - while (iter.hasNext) { - val key = iter.next() - close(key) - } + selector.close() } /** diff --git a/core/src/main/scala/kafka/network/Transmission.scala b/core/src/main/scala/kafka/network/Transmission.scala index 4c7a557..5176e34 100644 --- a/core/src/main/scala/kafka/network/Transmission.scala +++ b/core/src/main/scala/kafka/network/Transmission.scala @@ -54,7 +54,7 @@ abstract class MultiSend[S <: Send](val sends: List[S]) extends Send with Loggin private var current = sends var totalWritten = 0L - override def destination = -1; + override def destination = ""; override def reify = null; diff --git a/core/src/main/scala/kafka/producer/SyncProducer.scala b/core/src/main/scala/kafka/producer/SyncProducer.scala index 2a6c91e..6e5f4d3 100644 --- a/core/src/main/scala/kafka/producer/SyncProducer.scala +++ b/core/src/main/scala/kafka/producer/SyncProducer.scala @@ -51,7 +51,7 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { * data. So, leaving the rest of the logging at TRACE, while errors should be logged at ERROR level */ if (logger.isDebugEnabled) { - val buffer = new RequestOrResponseSend(request).buffer + val buffer = new RequestOrResponseSend("", request).buffer trace("verifying sendbuffer of size " + buffer.limit) val requestTypeId = buffer.getShort() if(requestTypeId == RequestKeys.ProduceKey) { diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index feea236..c46737d 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -108,7 +108,7 @@ class KafkaApis(val requestChannel: RequestChannel, try { val (response, error) = replicaManager.becomeLeaderOrFollower(leaderAndIsrRequest, offsetManager) val leaderAndIsrResponse = new LeaderAndIsrResponse(leaderAndIsrRequest.correlationId, response, error) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(leaderAndIsrResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, leaderAndIsrResponse))) } catch { case e: KafkaStorageException => fatal("Disk error during leadership change.", e) @@ -123,7 +123,7 @@ class KafkaApis(val requestChannel: RequestChannel, val stopReplicaRequest = request.requestObj.asInstanceOf[StopReplicaRequest] val (response, error) = replicaManager.stopReplicas(stopReplicaRequest) val stopReplicaResponse = new StopReplicaResponse(stopReplicaRequest.correlationId, response.toMap, error) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(stopReplicaResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, stopReplicaResponse))) replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads() } @@ -132,7 +132,7 @@ class KafkaApis(val requestChannel: RequestChannel, replicaManager.maybeUpdateMetadataCache(updateMetadataRequest, metadataCache) val updateMetadataResponse = new UpdateMetadataResponse(updateMetadataRequest.correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(updateMetadataResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, updateMetadataResponse))) } def handleControlledShutdownRequest(request: RequestChannel.Request) { @@ -143,7 +143,7 @@ class KafkaApis(val requestChannel: RequestChannel, val partitionsRemaining = controller.shutdownBroker(controlledShutdownRequest.brokerId) val controlledShutdownResponse = new ControlledShutdownResponse(controlledShutdownRequest.correlationId, ErrorMapping.NoError, partitionsRemaining) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(controlledShutdownResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, controlledShutdownResponse))) } @@ -167,7 +167,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val response = OffsetCommitResponse(commitStatus, offsetCommitRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } // compute the retention time based on the request version: @@ -243,7 +243,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } else { val response = ProducerResponse(produceRequest.correlationId, responseStatus) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } } @@ -288,7 +288,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val response = FetchResponse(fetchRequest.correlationId, responsePartitionData) - requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.requestKey.toString, response))) } // call the replica manager to fetch messages from the local replica @@ -346,7 +346,7 @@ class KafkaApis(val requestChannel: RequestChannel, } }) val response = OffsetResponse(offsetRequest.correlationId, responseMap) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } def fetchOffsets(logManager: LogManager, topicAndPartition: TopicAndPartition, timestamp: Long, maxNumOffsets: Int): Seq[Long] = { @@ -449,7 +449,7 @@ class KafkaApis(val requestChannel: RequestChannel, val brokers = metadataCache.getAliveBrokers trace("Sending topic metadata %s and brokers %s for correlation id %d to client %s".format(topicMetadata.mkString(","), brokers.mkString(","), metadataRequest.correlationId, metadataRequest.clientId)) val response = new TopicMetadataResponse(brokers.map(_.getBrokerEndPoint(request.securityProtocol)), topicMetadata, metadataRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } /* @@ -473,7 +473,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending offset fetch response %s for correlation id %d to client %s." .format(response, offsetFetchRequest.correlationId, offsetFetchRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } /* @@ -498,7 +498,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending consumer metadata %s for correlation id %d to client %s." .format(response, consumerMetadataRequest.correlationId, consumerMetadataRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) } def handleJoinGroupRequest(request: RequestChannel.Request) { diff --git a/core/src/main/scala/kafka/server/MessageSetSend.scala b/core/src/main/scala/kafka/server/MessageSetSend.scala index 484c5d8..d39f2f0 100644 --- a/core/src/main/scala/kafka/server/MessageSetSend.scala +++ b/core/src/main/scala/kafka/server/MessageSetSend.scala @@ -45,7 +45,7 @@ private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Sh def this() = this(MessageSet.Empty) - override def destination = -1 + override def destination = "" override def reify = null; diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 1b7d5d8..9973dad 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -12,14 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=OFF, stdout +log4j.rootLogger=TRACE, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n -log4j.logger.kafka=ERROR -log4j.logger.org.apache.kafka=ERROR +log4j.logger.kafka=TRACE +log4j.logger.org.apache.kafka=TRACE # zkclient can be verbose, during debugging it is common to adjust is separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN -- 2.3.2 (Apple Git-55) From 25f346052af5ffb410d2de87f83bc688e370872a Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 30 Apr 2015 17:47:38 -0700 Subject: [PATCH 06/24] renamed requestKey to connectionId to reflect new use and changed type from Any to String --- .../scala/kafka/api/ConsumerMetadataRequest.scala | 2 +- .../kafka/api/ControlledShutdownRequest.scala | 2 +- core/src/main/scala/kafka/api/FetchRequest.scala | 2 +- .../main/scala/kafka/api/LeaderAndIsrRequest.scala | 2 +- .../main/scala/kafka/api/OffsetCommitRequest.scala | 2 +- .../main/scala/kafka/api/OffsetFetchRequest.scala | 2 +- core/src/main/scala/kafka/api/OffsetRequest.scala | 2 +- .../src/main/scala/kafka/api/ProducerRequest.scala | 2 +- .../main/scala/kafka/api/StopReplicaRequest.scala | 2 +- .../scala/kafka/api/TopicMetadataRequest.scala | 2 +- .../scala/kafka/api/UpdateMetadataRequest.scala | 2 +- .../main/scala/kafka/network/RequestChannel.scala | 4 ++-- .../main/scala/kafka/network/SocketServer.scala | 6 ++--- core/src/main/scala/kafka/server/KafkaApis.scala | 28 +++++++++++----------- .../unit/kafka/network/SocketServerTest.scala | 2 +- 15 files changed, 31 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala index e221870..258d5fe 100644 --- a/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/ConsumerMetadataRequest.scala @@ -65,7 +65,7 @@ case class ConsumerMetadataRequest(group: String, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { // return ConsumerCoordinatorNotAvailable for all uncaught errors val errorResponse = ConsumerMetadataResponse(None, ErrorMapping.ConsumerCoordinatorNotAvailableCode, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } def describe(details: Boolean) = { diff --git a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala index b1e88ba..8092007 100644 --- a/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala +++ b/core/src/main/scala/kafka/api/ControlledShutdownRequest.scala @@ -62,7 +62,7 @@ case class ControlledShutdownRequest(versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = ControlledShutdownResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]]), Set.empty[TopicAndPartition]) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean = false): String = { diff --git a/core/src/main/scala/kafka/api/FetchRequest.scala b/core/src/main/scala/kafka/api/FetchRequest.scala index 033865e..5b38f85 100644 --- a/core/src/main/scala/kafka/api/FetchRequest.scala +++ b/core/src/main/scala/kafka/api/FetchRequest.scala @@ -149,7 +149,7 @@ case class FetchRequest(versionId: Short = FetchRequest.CurrentVersion, (topicAndPartition, FetchResponsePartitionData(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1, MessageSet.Empty)) } val errorResponse = FetchResponse(correlationId, fetchResponsePartitionData) - requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala index f8d92f2..c2584e0 100644 --- a/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala +++ b/core/src/main/scala/kafka/api/LeaderAndIsrRequest.scala @@ -186,7 +186,7 @@ case class LeaderAndIsrRequest (versionId: Short, case (topicAndPartition, partitionAndState) => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = LeaderAndIsrResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala index 5e4188e..84c2fa6 100644 --- a/core/src/main/scala/kafka/api/OffsetCommitRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetCommitRequest.scala @@ -163,7 +163,7 @@ case class OffsetCommitRequest(groupId: String, val commitStatus = requestInfo.mapValues(_ => errorCode) val commitResponse = OffsetCommitResponse(commitStatus, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, commitResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, commitResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala index 6186374..4bdeae9 100644 --- a/core/src/main/scala/kafka/api/OffsetFetchRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetFetchRequest.scala @@ -96,7 +96,7 @@ case class OffsetFetchRequest(groupId: String, )) }.toMap val errorResponse = OffsetFetchResponse(requestInfo=responseMap, correlationId=correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/OffsetRequest.scala b/core/src/main/scala/kafka/api/OffsetRequest.scala index 45c5c7d..f418868 100644 --- a/core/src/main/scala/kafka/api/OffsetRequest.scala +++ b/core/src/main/scala/kafka/api/OffsetRequest.scala @@ -118,7 +118,7 @@ case class OffsetRequest(requestInfo: Map[TopicAndPartition, PartitionOffsetRequ (topicAndPartition, PartitionOffsetsResponse(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), null)) } val errorResponse = OffsetResponse(correlationId, partitionOffsetResponseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/ProducerRequest.scala b/core/src/main/scala/kafka/api/ProducerRequest.scala index 41ff006..c866180 100644 --- a/core/src/main/scala/kafka/api/ProducerRequest.scala +++ b/core/src/main/scala/kafka/api/ProducerRequest.scala @@ -137,7 +137,7 @@ case class ProducerRequest(versionId: Short = ProducerRequest.CurrentVersion, (topicAndPartition, ProducerResponseStatus(ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]]), -1l)) } val errorResponse = ProducerResponse(correlationId, producerResponseStatus) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } } diff --git a/core/src/main/scala/kafka/api/StopReplicaRequest.scala b/core/src/main/scala/kafka/api/StopReplicaRequest.scala index ec914a9..4441fc6 100644 --- a/core/src/main/scala/kafka/api/StopReplicaRequest.scala +++ b/core/src/main/scala/kafka/api/StopReplicaRequest.scala @@ -106,7 +106,7 @@ case class StopReplicaRequest(versionId: Short, case topicAndPartition => (topicAndPartition, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) }.toMap val errorResponse = StopReplicaResponse(correlationId, responseMap) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala index e9afbf8..401c583 100644 --- a/core/src/main/scala/kafka/api/TopicMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/TopicMetadataRequest.scala @@ -82,7 +82,7 @@ case class TopicMetadataRequest(versionId: Short, topic => TopicMetadata(topic, Nil, ErrorMapping.codeFor(e.getClass.asInstanceOf[Class[Throwable]])) } val errorResponse = TopicMetadataResponse(Seq(), topicMetadata, correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala index 42cabcd..d59de82 100644 --- a/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala +++ b/core/src/main/scala/kafka/api/UpdateMetadataRequest.scala @@ -128,7 +128,7 @@ case class UpdateMetadataRequest (versionId: Short, override def handleError(e: Throwable, requestChannel: RequestChannel, request: RequestChannel.Request): Unit = { val errorResponse = new UpdateMetadataResponse(correlationId, ErrorMapping.codeFor(e.getCause.asInstanceOf[Class[Throwable]])) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, errorResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, errorResponse))) } override def describe(details: Boolean): String = { diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 9d7b2ef..4f75cbf 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -33,7 +33,7 @@ import org.apache.log4j.Logger object RequestChannel extends Logging { - val AllDone = new Request(processor = 1, requestKey = 2, buffer = getShutdownReceive(), startTimeMs = 0, securityProtocol = SecurityProtocol.PLAINTEXT) + val AllDone = new Request(processor = 1, connectionId = "2", buffer = getShutdownReceive(), startTimeMs = 0, securityProtocol = SecurityProtocol.PLAINTEXT) def getShutdownReceive() = { val emptyProducerRequest = new ProducerRequest(0, 0, "", 0, 0, collection.mutable.Map[TopicAndPartition, ByteBufferMessageSet]()) @@ -44,7 +44,7 @@ object RequestChannel extends Logging { byteBuffer } - case class Request(processor: Int, requestKey: Any, private var buffer: ByteBuffer, startTimeMs: Long, remoteAddress: SocketAddress = new InetSocketAddress(0), securityProtocol: SecurityProtocol) { + case class Request(processor: Int, connectionId: String, private var buffer: ByteBuffer, startTimeMs: Long, remoteAddress: SocketAddress = new InetSocketAddress(0), securityProtocol: SecurityProtocol) { @volatile var requestDequeueTimeMs = -1L @volatile var apiLocalCompleteTimeMs = -1L @volatile var responseCompleteTimeMs = -1L diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 505659f..4244c28 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -352,7 +352,7 @@ private[kafka] class Processor(val id: Int, val startSelectTime = new SystemTime().nanoseconds val ready = selector.poll(300) collection.JavaConversions.iterableAsScalaIterable(selector.completedReceives()).foreach( receive => { - val req = RequestChannel.Request(processor = id, requestKey = receive.source(), buffer = receive.payload(), startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) + val req = RequestChannel.Request(processor = id, connectionId = receive.source(), buffer = receive.payload(), startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) requestChannel.sendRequest(req) selector.mute(receive.source()) }) @@ -374,7 +374,7 @@ private[kafka] class Processor(val id: Int, // that are sitting in the server's socket buffer curr.request.updateRequestMetrics trace("Socket server received empty response to send, registering for read: " + curr) - selector.unmute(curr.request.requestKey.toString) + selector.unmute(curr.request.connectionId) } case RequestChannel.SendAction => { trace("Socket server received response to send, registering for write, sending data and registering for read: " + curr) @@ -384,7 +384,7 @@ private[kafka] class Processor(val id: Int, case RequestChannel.CloseConnectionAction => { curr.request.updateRequestMetrics trace("Closing socket connection actively according to the response code.") - selector.close(curr.request.requestKey.toString) + selector.close(curr.request.connectionId) } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 75f8596..dca1d0a 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -93,7 +93,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (response == null) requestChannel.closeConnection(request.processor, request) else - requestChannel.sendResponse(new Response(request, new ResponseSend(request.requestKey.toString, respHeader, response))) + requestChannel.sendResponse(new Response(request, new ResponseSend(request.connectionId, respHeader, response))) } error("error when handling request %s".format(request.requestObj), e) } finally @@ -108,7 +108,7 @@ class KafkaApis(val requestChannel: RequestChannel, try { val (response, error) = replicaManager.becomeLeaderOrFollower(leaderAndIsrRequest, offsetManager) val leaderAndIsrResponse = new LeaderAndIsrResponse(leaderAndIsrRequest.correlationId, response, error) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, leaderAndIsrResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, leaderAndIsrResponse))) } catch { case e: KafkaStorageException => fatal("Disk error during leadership change.", e) @@ -123,7 +123,7 @@ class KafkaApis(val requestChannel: RequestChannel, val stopReplicaRequest = request.requestObj.asInstanceOf[StopReplicaRequest] val (response, error) = replicaManager.stopReplicas(stopReplicaRequest) val stopReplicaResponse = new StopReplicaResponse(stopReplicaRequest.correlationId, response.toMap, error) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, stopReplicaResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, stopReplicaResponse))) replicaManager.replicaFetcherManager.shutdownIdleFetcherThreads() } @@ -132,7 +132,7 @@ class KafkaApis(val requestChannel: RequestChannel, replicaManager.maybeUpdateMetadataCache(updateMetadataRequest, metadataCache) val updateMetadataResponse = new UpdateMetadataResponse(updateMetadataRequest.correlationId) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, updateMetadataResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, updateMetadataResponse))) } def handleControlledShutdownRequest(request: RequestChannel.Request) { @@ -143,7 +143,7 @@ class KafkaApis(val requestChannel: RequestChannel, val partitionsRemaining = controller.shutdownBroker(controlledShutdownRequest.brokerId) val controlledShutdownResponse = new ControlledShutdownResponse(controlledShutdownRequest.correlationId, ErrorMapping.NoError, partitionsRemaining) - requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.requestKey.toString, controlledShutdownResponse))) + requestChannel.sendResponse(new Response(request, new RequestOrResponseSend(request.connectionId, controlledShutdownResponse))) } @@ -167,7 +167,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val response = OffsetCommitResponse(commitStatus, offsetCommitRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } // compute the retention time based on the request version: @@ -243,7 +243,7 @@ class KafkaApis(val requestChannel: RequestChannel, } } else { val response = ProducerResponse(produceRequest.correlationId, responseStatus) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } } @@ -288,7 +288,7 @@ class KafkaApis(val requestChannel: RequestChannel, } val response = FetchResponse(fetchRequest.correlationId, responsePartitionData) - requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new FetchResponseSend(request.connectionId, response))) } // call the replica manager to fetch messages from the local replica @@ -346,7 +346,7 @@ class KafkaApis(val requestChannel: RequestChannel, } }) val response = OffsetResponse(offsetRequest.correlationId, responseMap) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } def fetchOffsets(logManager: LogManager, topicAndPartition: TopicAndPartition, timestamp: Long, maxNumOffsets: Int): Seq[Long] = { @@ -449,7 +449,7 @@ class KafkaApis(val requestChannel: RequestChannel, val brokers = metadataCache.getAliveBrokers trace("Sending topic metadata %s and brokers %s for correlation id %d to client %s".format(topicMetadata.mkString(","), brokers.mkString(","), metadataRequest.correlationId, metadataRequest.clientId)) val response = new TopicMetadataResponse(brokers.map(_.getBrokerEndPoint(request.securityProtocol)), topicMetadata, metadataRequest.correlationId) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } /* @@ -473,7 +473,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending offset fetch response %s for correlation id %d to client %s." .format(response, offsetFetchRequest.correlationId, offsetFetchRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } /* @@ -498,7 +498,7 @@ class KafkaApis(val requestChannel: RequestChannel, trace("Sending consumer metadata %s for correlation id %d to client %s." .format(response, consumerMetadataRequest.correlationId, consumerMetadataRequest.clientId)) - requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.requestKey.toString, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new RequestOrResponseSend(request.connectionId, response))) } def handleJoinGroupRequest(request: RequestChannel.Request) { @@ -511,7 +511,7 @@ class KafkaApis(val requestChannel: RequestChannel, def sendResponseCallback(partitions: List[TopicAndPartition], generationId: Int, errorCode: Short) { val partitionList = partitions.map(tp => new TopicPartition(tp.topic, tp.partition)).toBuffer val responseBody = new JoinGroupResponse(errorCode, generationId, joinGroupRequest.consumerId, partitionList) - requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.requestKey.toString, respHeader, responseBody))) + requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, responseBody))) } // let the coordinator to handle join-group @@ -531,7 +531,7 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for sending a heartbeat response def sendResponseCallback(errorCode: Short) { val response = new HeartbeatResponse(errorCode) - requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.requestKey.toString, respHeader, response))) + requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, response))) } // let the coordinator to handle heartbeat diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 37267d9..8047f52 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -72,7 +72,7 @@ class SocketServerTest extends JUnitSuite { val byteBuffer = ByteBuffer.allocate(request.requestObj.sizeInBytes) request.requestObj.writeTo(byteBuffer) byteBuffer.rewind() - val send = new NetworkSend(request.requestKey.toString, byteBuffer) + val send = new NetworkSend(request.connectionId, byteBuffer) channel.sendResponse(new RequestChannel.Response(request.processor, request, send)) } -- 2.3.2 (Apple Git-55) From 05b187f004f5e1ae5f051a3abc91b177ed2c84fb Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Fri, 1 May 2015 15:45:17 -0700 Subject: [PATCH 07/24] Following Jun's comments - moved MultiSend to client. Cleaned up destinations as well --- .../org/apache/kafka/common/network/MultiSend.java | 101 +++++++++++++++++++++ .../kafka/common/network/NetworkReceive.java | 4 +- .../apache/kafka/common/network/NetworkSend.java | 2 - .../org/apache/kafka/common/network/Selector.java | 2 +- core/src/main/scala/kafka/api/FetchResponse.scala | 23 ++--- .../main/scala/kafka/network/Transmission.scala | 93 ------------------- core/src/main/scala/kafka/server/KafkaApis.scala | 9 -- .../main/scala/kafka/server/MessageSetSend.scala | 78 ---------------- 8 files changed, 114 insertions(+), 198 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/network/MultiSend.java delete mode 100644 core/src/main/scala/kafka/network/Transmission.scala delete mode 100644 core/src/main/scala/kafka/server/MessageSetSend.scala diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java new file mode 100644 index 0000000..862620c --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.network; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.GatheringByteChannel; +import java.util.List; +import java.util.ListIterator; + +/** + * A set of composite sends, sent one after another + */ + +public class MultiSend implements Send { + + private static final Logger log = LoggerFactory.getLogger(MultiSend.class); + private static String dest; + private static List sends; + private static long expectedBytesToWrite; + private long totalWritten = 0; + private ListIterator sendListIterator; + + //private int current; + //private var current = sends + //public static long totalWritten = 0; + + public MultiSend(String dest, List sends, long expectedBytesToWrite) { + this.dest = dest; + this.sends = sends; + this.expectedBytesToWrite = expectedBytesToWrite; + sendListIterator = sends.listIterator(); + } + + @Override + public String destination() { + return dest; + } + + @Override + public long remaining() { + return expectedBytesToWrite - totalWritten; + } + + @Override + public boolean completed() { + if (!sendListIterator.hasNext()) { + if (totalWritten != expectedBytesToWrite) + log.error("mismatch in sending bytes over socket; expected: " + expectedBytesToWrite + " actual: " + totalWritten); + return true; + } else { + return false; + } + } + + @Override + public ByteBuffer[] reify() { + return null; + } + + @Override + public long writeTo(GatheringByteChannel channel) throws IOException { + //TODO: Is this needed? expectIncomplete + long totalWrittenPerCall = 0; + boolean sendComplete = false; + + do { + Send current = sendListIterator.next(); + long written = current.writeTo(channel); + totalWritten += written; + totalWrittenPerCall += written; + sendComplete = current.completed(); + if (!sendComplete) { + // move iterator back so we will keep processing same send + sendListIterator.previous(); + } + + } while (!completed() && sendComplete); + if (log.isTraceEnabled()) { + log.trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + expectedBytesToWrite); + } + return totalWrittenPerCall; + } +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index f493087..e97d70e 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -28,7 +28,7 @@ public class NetworkReceive implements Receive { private final String source; private final ByteBuffer size; - private final long maxSize; + private final int maxSize; private ByteBuffer buffer; @@ -46,7 +46,7 @@ public class NetworkReceive implements Receive { this.maxSize = UNLIMITED; } - public NetworkReceive(long maxSize, String source) { + public NetworkReceive(int maxSize, String source) { this.source = source; this.size = ByteBuffer.allocate(4); this.buffer = null; diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java index 339e8ae..49964b0 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkSend.java @@ -23,8 +23,6 @@ import java.nio.ByteBuffer; */ public class NetworkSend extends ByteBufferSend { - public final static String UNKNOWN_DEST = ""; - public NetworkSend(String destination, ByteBuffer... buffers) { super(destination, sizeDelimit(buffers)); } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 1a5fdef..4e6b20d 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -81,7 +81,7 @@ public class Selector implements Selectable { private final Map metricTags; private final Map lruConnections; private long currentTimeNanos; - private long maxReceiveSize; + private int maxReceiveSize; /** diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 203ffc3..0d1a59d 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -22,9 +22,9 @@ import java.nio.channels.GatheringByteChannel import kafka.common.{TopicAndPartition, ErrorMapping} import kafka.message.{MessageSet, ByteBufferMessageSet} -import kafka.network.MultiSend import kafka.api.ApiUtils._ import org.apache.kafka.common.network.Send +import org.apache.kafka.common.network.MultiSend import scala.collection._ @@ -108,15 +108,16 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti val headerSize = TopicData.headerSize(topic) } -class TopicDataSend(val topicData: TopicData) extends Send { +class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { private val size = topicData.sizeInBytes private var sent = 0L override def completed: Boolean = sent >= size - override def destination: String = "" + override def destination: String = dest + // There are no buffers to return override def reify = null override def remaining: Long = (size.toLong - sent) @@ -126,10 +127,8 @@ class TopicDataSend(val topicData: TopicData) extends Send { buffer.putInt(topicData.partitionData.size) buffer.rewind() - val sends = new MultiSend(topicData.partitionData.toList - .map(d => new PartitionDataSend(d._1, d._2))) { - val expectedBytesToWrite = topicData.sizeInBytes - topicData.headerSize - } + val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(topicData.partitionData.toList + .map(d => new PartitionDataSend(d._1, d._2))), topicData.sizeInBytes - topicData.headerSize) def writeTo(channel: GatheringByteChannel): Long = { // TODO: expectIncomplete() @@ -234,14 +233,12 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte buffer.putInt(fetchResponse.dataGroupedByTopic.size) // topic count buffer.rewind() - val sends = new MultiSend(fetchResponse.dataGroupedByTopic.toList.map { - case(topic, data) => new TopicDataSend(TopicData(topic, + val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(fetchResponse.dataGroupedByTopic.toList.map { + case(topic, data) => new TopicDataSend(dest, TopicData(topic, data.map{case(topicAndPartition, message) => (topicAndPartition.partition, message)})) - }) { - val expectedBytesToWrite = fetchResponse.sizeInBytes - FetchResponse.headerSize - } + }), fetchResponse.sizeInBytes - FetchResponse.headerSize) - def writeTo(channel: GatheringByteChannel):Long = { + def writeTo(channel: GatheringByteChannel): Long = { //TODO: expectIncomplete() var written = 0L if(buffer.hasRemaining) diff --git a/core/src/main/scala/kafka/network/Transmission.scala b/core/src/main/scala/kafka/network/Transmission.scala deleted file mode 100644 index 5176e34..0000000 --- a/core/src/main/scala/kafka/network/Transmission.scala +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.network - -import java.nio._ -import java.nio.channels._ -import kafka.utils.Logging -import kafka.common.KafkaException -import org.apache.kafka.common.network.Send - -/** - * Represents a stateful transfer of data to or from the network - */ -private[network] trait Transmission extends Logging { - - def complete: Boolean - - protected def expectIncomplete(): Unit = { - if(complete) - throw new KafkaException("This operation cannot be completed on a complete request.") - } - - protected def expectComplete(): Unit = { - if(!complete) - throw new KafkaException("This operation cannot be completed on an incomplete request.") - } - -} - - - - - -/** - * A set of composite sends, sent one after another - */ -abstract class MultiSend[S <: Send](val sends: List[S]) extends Send with Logging { - val expectedBytesToWrite: Int - private var current = sends - var totalWritten = 0L - - override def destination = ""; - - override def reify = null; - - override def remaining = (expectedBytesToWrite - totalWritten) - - /** - * This method continues to write to the socket buffer till an incomplete - * write happens. On an incomplete write, it returns to the caller to give it - * a chance to schedule other work till the buffered write completes. - */ - def writeTo(channel: GatheringByteChannel): Long = { - //TODO: Is this needed? expectIncomplete - var totalWrittenPerCall = 0L - var sendComplete: Boolean = false - do { - val written = current.head.writeTo(channel) - totalWritten += written - totalWrittenPerCall += written - sendComplete = current.head.completed() - if(sendComplete) - current = current.tail - } while (!completed && sendComplete) - trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + expectedBytesToWrite) - totalWrittenPerCall - } - - def completed: Boolean = { - if (current == Nil) { - if (totalWritten != expectedBytesToWrite) - error("mismatch in sending bytes over socket; expected: " + expectedBytesToWrite + " actual: " + totalWritten) - true - } else { - false - } - } -} diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index dca1d0a..047cd13 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,15 +17,6 @@ package kafka.server - -import kafka.api.ConsumerMetadataRequest -import kafka.api.ConsumerMetadataResponse -import kafka.api.FetchRequest -import kafka.api.FetchResponse -import kafka.api.OffsetCommitRequest -import kafka.api.OffsetCommitResponse -import kafka.api.OffsetFetchRequest -import kafka.api.OffsetFetchResponse import org.apache.kafka.common.protocol.SecurityProtocol import org.apache.kafka.common.TopicPartition import kafka.api._ diff --git a/core/src/main/scala/kafka/server/MessageSetSend.scala b/core/src/main/scala/kafka/server/MessageSetSend.scala deleted file mode 100644 index d39f2f0..0000000 --- a/core/src/main/scala/kafka/server/MessageSetSend.scala +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server - -import java.nio._ -import java.nio.channels._ -import kafka.network._ -import kafka.message._ -import kafka.utils._ -import kafka.common.ErrorMapping -import org.apache.kafka.common.network.Send - -/** - * A zero-copy message response that writes the bytes needed directly from the file - * wholly in kernel space - */ -@nonthreadsafe -private[server] class MessageSetSend(val messages: MessageSet, val errorCode: Short) extends Send with Logging{ - - private var sent: Int = 0 - private val size: Int = messages.sizeInBytes - private val header = ByteBuffer.allocate(6) - header.putInt(size + 2) - header.putShort(errorCode) - header.rewind() - - var completed: Boolean = false - - def this(messages: MessageSet) = this(messages, ErrorMapping.NoError) - - def this() = this(MessageSet.Empty) - - override def destination = "" - - override def reify = null; - - override def remaining = size - sent - - def writeTo(channel: GatheringByteChannel): Long = { - //TODO: expectIncomplete() - var written = 0 - if(header.hasRemaining) - written += channel.write(header) - if(!header.hasRemaining) { - val fileBytesSent = messages.writeTo(channel, sent, size - sent) - written += fileBytesSent - sent += fileBytesSent - } - - if(logger.isTraceEnabled) - if (channel.isInstanceOf[SocketChannel]) { - val socketChannel = channel.asInstanceOf[SocketChannel] - logger.trace(sent + " bytes written to " + socketChannel.socket.getRemoteSocketAddress() + " expecting to send " + size + " bytes") - } - - if(sent >= size) - completed = true - written - } - - def sendSize: Int = size + header.capacity - -} -- 2.3.2 (Apple Git-55) From b446592cce8dba8cc36f8251836071e99a9b07d4 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Tue, 12 May 2015 11:38:54 +0300 Subject: [PATCH 08/24] removed reify and remaining from send/recieve API, per Jun. moved maybeCloseOldest() to Selector per Jay --- .../kafka/clients/consumer/ConsumerConfig.java | 11 +++++- .../kafka/clients/consumer/KafkaConsumer.java | 3 +- .../kafka/clients/producer/KafkaProducer.java | 3 +- .../kafka/clients/producer/ProducerConfig.java | 7 +++- .../kafka/common/network/ByteBufferReceive.java | 4 -- .../kafka/common/network/ByteBufferSend.java | 10 ----- .../org/apache/kafka/common/network/MultiSend.java | 11 ------ .../kafka/common/network/NetworkReceive.java | 6 --- .../org/apache/kafka/common/network/Receive.java | 6 --- .../org/apache/kafka/common/network/Selector.java | 44 ++++++++++++---------- .../java/org/apache/kafka/common/network/Send.java | 11 ------ .../apache/kafka/common/network/SelectorTest.java | 2 +- core/src/main/scala/kafka/api/FetchResponse.scala | 13 ------- .../main/scala/kafka/network/SocketServer.scala | 13 +------ 14 files changed, 48 insertions(+), 96 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index bdff518..55837cc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -156,6 +156,10 @@ public class ConsumerConfig extends AbstractConfig { public static final String VALUE_DESERIALIZER_CLASS_CONFIG = "value.deserializer"; private static final String VALUE_DESERIALIZER_CLASS_DOC = "Deserializer class for value that implements the Deserializer interface."; + /** connections.max.idle.ms */ + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; + public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, @@ -277,7 +281,12 @@ public class ConsumerConfig extends AbstractConfig { .define(VALUE_DESERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, - VALUE_DESERIALIZER_CLASS_DOC); + VALUE_DESERIALIZER_CLASS_DOC) + .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, + Type.LONG, + 10 * 60 * 1000, + Importance.MEDIUM, + CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addDeserializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index d301be4..d1d1ec1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -472,7 +472,8 @@ public class KafkaConsumer implements Consumer { String metricGrpPrefix = "consumer"; Map metricsTags = new LinkedHashMap(); metricsTags.put("client-id", clientId); - this.client = new NetworkClient(new Selector(metrics, time, metricGrpPrefix, metricsTags), + this.client = new NetworkClient( + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, metricsTags), this.metadata, clientId, 100, // a fixed large enough value will suffice diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 42b1292..43e7123 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -228,7 +228,8 @@ public class KafkaProducer implements Producer { List addresses = ClientUtils.parseAndValidateAddresses(config.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)); this.metadata.update(Cluster.bootstrap(addresses), time.milliseconds()); - NetworkClient client = new NetworkClient(new Selector(this.metrics, time, "producer", metricTags), + NetworkClient client = new NetworkClient( + new Selector(config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), this.metrics, time, "producer", metricTags), this.metadata, clientId, config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index 187d000..7145fb0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -169,6 +169,10 @@ public class ProducerConfig extends AbstractConfig { public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that implements the Serializer interface."; + /** connections.max.idle.ms */ + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; + public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOSTRAP_SERVERS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, 32 * 1024 * 1024L, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) @@ -217,7 +221,8 @@ public class ProducerConfig extends AbstractConfig { Importance.LOW, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DOC) .define(KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) - .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC); + .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, 10 * 60 * 1000, Importance.MEDIUM, CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addSerializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java index cbb2f14..159c301 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferReceive.java @@ -54,8 +54,4 @@ public class ByteBufferReceive implements Receive { return read; } - public ByteBuffer[] reify() { - return buffers; - } - } diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index 45fadfc..91e975e 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -46,16 +46,6 @@ public class ByteBufferSend implements Send { return remaining <= 0; } - @Override - public ByteBuffer[] reify() { - return this.buffers; - } - - @Override - public long remaining() { - return this.remaining; - } - public int size() { return this.size; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index 862620c..7b86277 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.GatheringByteChannel; import java.util.List; import java.util.ListIterator; @@ -55,11 +54,6 @@ public class MultiSend implements Send { } @Override - public long remaining() { - return expectedBytesToWrite - totalWritten; - } - - @Override public boolean completed() { if (!sendListIterator.hasNext()) { if (totalWritten != expectedBytesToWrite) @@ -71,11 +65,6 @@ public class MultiSend implements Send { } @Override - public ByteBuffer[] reify() { - return null; - } - - @Override public long writeTo(GatheringByteChannel channel) throws IOException { //TODO: Is this needed? expectIncomplete long totalWrittenPerCall = 0; diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index e97d70e..c3d9db0 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -67,12 +67,6 @@ public class NetworkReceive implements Receive { return !size.hasRemaining() && !buffer.hasRemaining(); } - @Override - public ByteBuffer[] reify() { - return new ByteBuffer[] {this.buffer}; - } - - public long readFrom(ScatteringByteChannel channel) throws IOException { return readFromReadableChannel(channel); } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Receive.java b/clients/src/main/java/org/apache/kafka/common/network/Receive.java index ac4e3f9..4b14431 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Receive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Receive.java @@ -17,7 +17,6 @@ package org.apache.kafka.common.network; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.ScatteringByteChannel; /** @@ -36,11 +35,6 @@ public interface Receive { public boolean complete(); /** - * Turn this receive into ByteBuffer instances, if possible (otherwise returns null). - */ - public ByteBuffer[] reify(); - - /** * Read bytes into this receive from the given channel * @param channel The channel to read from * @return The number of bytes read diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 4e6b20d..f79f12b 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -80,20 +80,23 @@ public class Selector implements Selectable { private final String metricGrpPrefix; private final Map metricTags; private final Map lruConnections; + private final long connectionsMaxIdleNanos; + private final int maxReceiveSize; private long currentTimeNanos; - private int maxReceiveSize; + private long nextIdleCloseCheckTime; /** * Create a new nioSelector */ - public Selector(int maxReceiveSize, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { + public Selector(int maxReceiveSize, long connectionMaxIdleMs, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { try { this.nioSelector = java.nio.channels.Selector.open(); } catch (IOException e) { throw new KafkaException(e); } this.maxReceiveSize = maxReceiveSize; + this.connectionsMaxIdleNanos = connectionMaxIdleMs * 1000 * 1000; this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; @@ -106,10 +109,11 @@ public class Selector implements Selectable { this.sensors = new SelectorMetrics(metrics); this.lruConnections = new LinkedHashMap(); currentTimeNanos = new SystemTime().nanoseconds(); + nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; } - public Selector(Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(-1, metrics, time, metricGrpPrefix, metricTags); + public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { + this(-1, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags); } /** @@ -277,7 +281,7 @@ public class Selector implements Selectable { /* write to any sockets that have space in their buffer and for which we have data */ if (key.isWritable()) { transmissions.send.writeTo(channel); - if (transmissions.send.remaining() <= 0) { + if (transmissions.send.completed()) { this.completedSends.add(transmissions.send); // TODO: Figure out the metrics here. Send doesn't have "size" in general and the server-side records sent messages, not sent bytes. this.sensors.recordBytesSent(transmissions.id, 0); @@ -309,6 +313,7 @@ public class Selector implements Selectable { } long endIo = time.nanoseconds(); this.sensors.ioTime.record(endIo - endSelect, time.milliseconds()); + maybeCloseOldestConnection(); } private String socketDescription(SocketChannel channel) { @@ -371,21 +376,22 @@ public class Selector implements Selectable { unmute(key); } - public long maybeCloseOldestConnection(long connectionsMaxIdleNanos) { - if (lruConnections.isEmpty()) { - return currentTimeNanos + connectionsMaxIdleNanos; - } else { - Map.Entry oldestConnectionEntry = lruConnections.entrySet().iterator().next(); - Long connectionLastActiveTime = oldestConnectionEntry.getValue(); - Long nextIdleCloseCheckTime = connectionLastActiveTime + connectionsMaxIdleNanos; - if (currentTimeNanos > nextIdleCloseCheckTime) { - String connectionId = oldestConnectionEntry.getKey(); - if (log.isTraceEnabled()) - log.trace("About to close the idle connection from " + connectionId - + " due to being idle for " + (currentTimeNanos - connectionLastActiveTime) / 1000 / 1000 + " millis"); - close(connectionId); + private void maybeCloseOldestConnection() { + if (currentTimeNanos > nextIdleCloseCheckTime) { + if (lruConnections.isEmpty()) { + nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; + } else { + Map.Entry oldestConnectionEntry = lruConnections.entrySet().iterator().next(); + Long connectionLastActiveTime = oldestConnectionEntry.getValue(); + nextIdleCloseCheckTime = connectionLastActiveTime + connectionsMaxIdleNanos; + if (currentTimeNanos > nextIdleCloseCheckTime) { + String connectionId = oldestConnectionEntry.getKey(); + if (log.isTraceEnabled()) + log.trace("About to close the idle connection from " + connectionId + + " due to being idle for " + (currentTimeNanos - connectionLastActiveTime) / 1000 / 1000 + " millis"); + close(connectionId); + } } - return nextIdleCloseCheckTime; } } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index a94d9c3..9a09716 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -13,7 +13,6 @@ package org.apache.kafka.common.network; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.channels.GatheringByteChannel; /** @@ -27,21 +26,11 @@ public interface Send { public String destination(); /** - * The number of bytes remaining to send - */ - public long remaining(); - - /** * Is this send complete? */ public boolean completed(); /** - * An optional method to turn this send into an array of ByteBuffers if possible (otherwise returns null) - */ - public ByteBuffer[] reify(); - - /** * Write some as-yet unwritten bytes from this send to the provided channel. It may take multiple calls for the send * to be completely written * @param channel The channel to write to diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 5a49975..d23b4b6 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -46,7 +46,7 @@ public class SelectorTest { public void setup() throws Exception { this.server = new EchoServer(); this.server.start(); - this.selector = new Selector(new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); + this.selector = new Selector(5000, new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); } @After diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 0d1a59d..2f8be3b 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -67,10 +67,6 @@ class PartitionDataSend(val partitionId: Int, override def destination: String = "" - override def reify = null - - override def remaining: Long = buffer.remaining().toLong + (messageSize - messagesSentSize) - override def writeTo(channel: GatheringByteChannel): Long = { var written = 0 if(buffer.hasRemaining) @@ -117,11 +113,6 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { override def destination: String = dest - // There are no buffers to return - override def reify = null - - override def remaining: Long = (size.toLong - sent) - private val buffer = ByteBuffer.allocate(topicData.headerSize) writeShortString(buffer, topicData.topic) buffer.putInt(topicData.partitionData.size) @@ -223,10 +214,6 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte override def destination = dest - override def reify = null - - override def remaining = sendSize - sent - private val buffer = ByteBuffer.allocate(4 /* for size */ + FetchResponse.headerSize) buffer.putInt(size) buffer.putInt(fetchResponse.correlationId) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 4244c28..3ff1dd1 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -327,15 +327,14 @@ private[kafka] class Processor(val id: Int, val portToProtocol: ConcurrentHashMap[Int,SecurityProtocol]) extends AbstractServerThread(connectionQuotas) { private val newConnections = new ConcurrentLinkedQueue[SocketChannel]() - private val connectionsMaxIdleNanos = connectionsMaxIdleMs * 1000 * 1000 - private var currentTimeNanos = new SystemTime().nanoseconds private val lruConnections = new util.LinkedHashMap[SelectionKey, Long] - private var nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos + private val metrics = new Metrics(time); private var metricTags = new util.HashMap[String, String]() metricTags.put("processor-id", id.toString) private val selector = new org.apache.kafka.common.network.Selector( maxRequestSize, + connectionsMaxIdleMs, metrics, time, "SocketServer", @@ -356,8 +355,6 @@ private[kafka] class Processor(val id: Int, requestChannel.sendRequest(req) selector.mute(receive.source()) }) - - maybeCloseOldestConnection } debug("Closing selector - processor " + id) closeAll() @@ -419,12 +416,6 @@ private[kafka] class Processor(val id: Int, } } - private def maybeCloseOldestConnection { - if(currentTimeNanos > nextIdleCloseCheckTime) { - nextIdleCloseCheckTime = selector.maybeCloseOldestConnection(connectionsMaxIdleNanos) - } - } - /** * Close all open connections */ -- 2.3.2 (Apple Git-55) From 675f3cb8a19063e329cd4d76526d8285530e0063 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Tue, 12 May 2015 12:57:52 +0300 Subject: [PATCH 09/24] added idString to node API, changed written to int in Send API --- .../org/apache/kafka/clients/NetworkClient.java | 32 +++++++++++----------- .../clients/consumer/internals/Coordinator.java | 4 +-- .../kafka/clients/consumer/internals/Fetcher.java | 6 ++-- .../main/java/org/apache/kafka/common/Node.java | 10 +++++++ .../kafka/common/network/ByteBufferSend.java | 6 ++-- .../org/apache/kafka/common/network/MultiSend.java | 6 ++-- .../java/org/apache/kafka/common/network/Send.java | 2 +- .../apache/kafka/clients/NetworkClientTest.java | 6 ++-- core/src/main/scala/kafka/api/FetchResponse.scala | 14 +++++----- 9 files changed, 49 insertions(+), 37 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index b0d4270..0e71ba5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -111,7 +111,7 @@ public class NetworkClient implements KafkaClient { if (isReady(node, now)) return true; - if (connectionStates.canConnect(Integer.toString(node.id()), now)) + if (connectionStates.canConnect(node.idString(), now)) // if we are interested in sending to a node and we don't have a connection to it, initiate one initiateConnect(node, now); @@ -129,7 +129,7 @@ public class NetworkClient implements KafkaClient { */ @Override public long connectionDelay(Node node, long now) { - return connectionStates.connectionDelay(Integer.toString(node.id()), now); + return connectionStates.connectionDelay(node.idString(), now); } /** @@ -142,7 +142,7 @@ public class NetworkClient implements KafkaClient { */ @Override public boolean connectionFailed(Node node) { - return connectionStates.connectionState(Integer.toString(node.id())).equals(ConnectionState.DISCONNECTED); + return connectionStates.connectionState(node.idString()).equals(ConnectionState.DISCONNECTED); } /** @@ -154,14 +154,14 @@ public class NetworkClient implements KafkaClient { */ @Override public boolean isReady(Node node, long now) { - int nodeId = node.id(); + String nodeId = node.idString(); if (!this.metadataFetchInProgress && this.metadata.timeToNextUpdate(now) == 0) // if we need to update our metadata now declare all requests unready to make metadata requests first // priority return false; else // otherwise we are ready if we are connected and can send more requests - return isSendable(Integer.toString(nodeId)); + return isSendable(nodeId); } /** @@ -288,8 +288,8 @@ public class NetworkClient implements KafkaClient { * Get the number of in-flight requests for a given node */ @Override - public int inFlightRequestCount(String nodeId) { - return this.inFlightRequests.inFlightRequestCount(nodeId); + public int inFlightRequestCount(String node) { + return this.inFlightRequests.inFlightRequestCount(node); } /** @@ -334,11 +334,11 @@ public class NetworkClient implements KafkaClient { for (int i = 0; i < nodes.size(); i++) { int idx = Utils.abs((this.nodeIndexOffset + i) % nodes.size()); Node node = nodes.get(idx); - int currInflight = this.inFlightRequests.inFlightRequestCount(Integer.toString(node.id())); - if (currInflight == 0 && this.connectionStates.isConnected(Integer.toString(node.id()))) { + int currInflight = this.inFlightRequests.inFlightRequestCount(node.idString()); + if (currInflight == 0 && this.connectionStates.isConnected(node.idString())) { // if we find an established connection with no in-flight requests we can stop right away return node; - } else if (!this.connectionStates.isBlackedOut(Integer.toString(node.id()), now) && currInflight < inflight) { + } else if (!this.connectionStates.isBlackedOut(node.idString(), now) && currInflight < inflight) { // otherwise if this is the best we have found so far, record that inflight = currInflight; found = node; @@ -429,9 +429,9 @@ public class NetworkClient implements KafkaClient { * Record any newly completed connections */ private void handleConnections() { - for (String id : this.selector.connected()) { - log.debug("Completed connection to node {}", id); - this.connectionStates.connected(id); + for (String node : this.selector.connected()) { + log.debug("Completed connection to node {}", node); + this.connectionStates.connected(node); } } @@ -466,7 +466,7 @@ public class NetworkClient implements KafkaClient { this.lastNoNodeAvailableMs = now; return; } - String nodeConnectionId = Integer.toString(node.id()); + String nodeConnectionId = node.idString(); if (connectionStates.isConnected(nodeConnectionId) && inFlightRequests.canSendMore(nodeConnectionId)) { @@ -495,11 +495,11 @@ public class NetworkClient implements KafkaClient { * Initiate a connection to the given node */ private void initiateConnect(Node node, long now) { - String nodeConnectionId = Integer.toString(node.id()); + String nodeConnectionId = node.idString(); try { log.debug("Initiating connection to node {} at {}:{}.", node.id(), node.host(), node.port()); this.connectionStates.connecting(nodeConnectionId, now); - selector.connect(Integer.toString(node.id()), + selector.connect(nodeConnectionId, new InetSocketAddress(node.host(), node.port()), this.socketSendBuffer, this.socketReceiveBuffer); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java index 637717b..27ff351 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Coordinator.java @@ -416,7 +416,7 @@ public final class Coordinator { log.debug("Issuing consumer metadata request to broker {}", node.id()); ConsumerMetadataRequest request = new ConsumerMetadataRequest(this.groupId); - RequestSend send = new RequestSend(Integer.toString(node.id()), + RequestSend send = new RequestSend(node.idString(), this.client.nextRequestHeader(ApiKeys.CONSUMER_METADATA), request.toStruct()); long now = time.milliseconds(); @@ -435,7 +435,7 @@ public final class Coordinator { log.debug("Issuing request ({}: {}) to coordinator {}", api, request, this.consumerCoordinator.id()); RequestHeader header = this.client.nextRequestHeader(api); - RequestSend send = new RequestSend(Integer.toString(this.consumerCoordinator.id()), header, request); + RequestSend send = new RequestSend(this.consumerCoordinator.idString(), header, request); return new ClientRequest(now, true, send, handler); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 01188be..94f5a66 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -209,12 +209,12 @@ public class Fetcher { } else if (this.client.ready(info.leader(), now)) { Node node = info.leader(); ListOffsetRequest request = new ListOffsetRequest(-1, partitions); - RequestSend send = new RequestSend(Integer.toString(node.id()), + RequestSend send = new RequestSend(node.idString(), this.client.nextRequestHeader(ApiKeys.LIST_OFFSETS), request.toStruct()); ClientRequest clientRequest = new ClientRequest(now, true, send, null); this.client.send(clientRequest); - List responses = this.client.completeAll(Integer.toString(node.id()), now); + List responses = this.client.completeAll(node.idString(), now); if (responses.isEmpty()) throw new IllegalStateException("This should not happen."); ClientResponse response = responses.get(responses.size() - 1); @@ -257,7 +257,7 @@ public class Fetcher { for (TopicPartition partition : subscriptions.assignedPartitions()) { Node node = cluster.leaderFor(partition); // if there is a leader and no in-flight requests, issue a new fetch - if (node != null && this.client.inFlightRequestCount(Integer.toString(node.id())) == 0) { + if (node != null && this.client.inFlightRequestCount(node.idString()) == 0) { Map fetch = fetchable.get(node.id()); if (fetch == null) { fetch = new HashMap(); diff --git a/clients/src/main/java/org/apache/kafka/common/Node.java b/clients/src/main/java/org/apache/kafka/common/Node.java index f4e4186..644cd71 100644 --- a/clients/src/main/java/org/apache/kafka/common/Node.java +++ b/clients/src/main/java/org/apache/kafka/common/Node.java @@ -18,12 +18,14 @@ package org.apache.kafka.common; public class Node { private final int id; + private final String idString; private final String host; private final int port; public Node(int id, String host, int port) { super(); this.id = id; + this.idString = Integer.toString(id); this.host = host; this.port = port; } @@ -40,6 +42,14 @@ public class Node { } /** + * String representation of the node id. + * Typically the integer id is used to serialize over the wire, the string representation is used as an identifier with NetworkClient code + */ + public String idString() { + return idString; + } + + /** * The host name for this node */ public String host() { diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index 91e975e..0c13a7d 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -51,12 +51,14 @@ public class ByteBufferSend implements Send { } @Override - public long writeTo(GatheringByteChannel channel) throws IOException { + public int writeTo(GatheringByteChannel channel) throws IOException { long written = channel.write(buffers); if (written < 0) throw new EOFException("This shouldn't happen."); + if (written > Integer.MAX_VALUE) + throw new IOException("Wrote more bytes than " + size + ". This shouldn't happen."); remaining -= written; - return written; + return (int) written; } } diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index 7b86277..c122de2 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -65,14 +65,14 @@ public class MultiSend implements Send { } @Override - public long writeTo(GatheringByteChannel channel) throws IOException { + public int writeTo(GatheringByteChannel channel) throws IOException { //TODO: Is this needed? expectIncomplete - long totalWrittenPerCall = 0; + int totalWrittenPerCall = 0; boolean sendComplete = false; do { Send current = sendListIterator.next(); - long written = current.writeTo(channel); + int written = current.writeTo(channel); totalWritten += written; totalWrittenPerCall += written; sendComplete = current.completed(); diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index 9a09716..75f5001 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -37,6 +37,6 @@ public interface Send { * @return The number of bytes written * @throws IOException If the write fails */ - public long writeTo(GatheringByteChannel channel) throws IOException; + public int writeTo(GatheringByteChannel channel) throws IOException; } diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index cfc1303..43238ce 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -65,7 +65,7 @@ public class NetworkClientTest { client.poll(1, time.milliseconds()); selector.clear(); assertTrue("Now the client is ready", client.ready(node, time.milliseconds())); - selector.disconnect(Integer.toString(node.id())); + selector.disconnect(node.idString()); client.poll(1, time.milliseconds()); selector.clear(); assertFalse("After we forced the disconnection the client is no longer ready.", client.ready(node, time.milliseconds())); @@ -86,7 +86,7 @@ public class NetworkClientTest { public void testSimpleRequestResponse() { ProduceRequest produceRequest = new ProduceRequest((short) 1, 1000, Collections.emptyMap()); RequestHeader reqHeader = client.nextRequestHeader(ApiKeys.PRODUCE); - RequestSend send = new RequestSend(Integer.toString(node.id()), reqHeader, produceRequest.toStruct()); + RequestSend send = new RequestSend(node.idString(), reqHeader, produceRequest.toStruct()); TestCallbackHandler handler = new TestCallbackHandler(); ClientRequest request = new ClientRequest(time.milliseconds(), true, send, handler); awaitReady(client, node); @@ -101,7 +101,7 @@ public class NetworkClientTest { respHeader.writeTo(buffer); resp.writeTo(buffer); buffer.flip(); - selector.completeReceive(new NetworkReceive(Integer.toString(node.id()), buffer)); + selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List responses = client.poll(1, time.milliseconds()); assertEquals(1, responses.size()); assertTrue("The handler should have executed.", handler.executed); diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 2f8be3b..6d2130e 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -67,7 +67,7 @@ class PartitionDataSend(val partitionId: Int, override def destination: String = "" - override def writeTo(channel: GatheringByteChannel): Long = { + override def writeTo(channel: GatheringByteChannel): Int = { var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) @@ -107,7 +107,7 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { private val size = topicData.sizeInBytes - private var sent = 0L + private var sent = 0 override def completed: Boolean = sent >= size @@ -121,9 +121,9 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(topicData.partitionData.toList .map(d => new PartitionDataSend(d._1, d._2))), topicData.sizeInBytes - topicData.headerSize) - def writeTo(channel: GatheringByteChannel): Long = { + def writeTo(channel: GatheringByteChannel): Int = { // TODO: expectIncomplete() - var written = 0L + var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) if(!buffer.hasRemaining && !sends.completed) { @@ -206,7 +206,7 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { private val size = fetchResponse.sizeInBytes - private var sent = 0L + private var sent = 0 private val sendSize = 4 /* for size */ + size @@ -225,9 +225,9 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte data.map{case(topicAndPartition, message) => (topicAndPartition.partition, message)})) }), fetchResponse.sizeInBytes - FetchResponse.headerSize) - def writeTo(channel: GatheringByteChannel): Long = { + def writeTo(channel: GatheringByteChannel): Int = { //TODO: expectIncomplete() - var written = 0L + var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) if(!buffer.hasRemaining && !sends.completed) { -- 2.3.2 (Apple Git-55) From cba8152bf04514fbe6bae8838f9f78bc125b7b44 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 14 May 2015 13:53:26 +0300 Subject: [PATCH 10/24] cleaning up MultiSend, added size() to Send interface --- .../kafka/common/network/ByteBufferSend.java | 2 +- .../org/apache/kafka/common/network/MultiSend.java | 58 ++++++++++++---------- .../kafka/common/network/NetworkReceive.java | 16 +++--- .../org/apache/kafka/common/network/Selector.java | 10 ++-- .../java/org/apache/kafka/common/network/Send.java | 5 ++ core/src/main/scala/kafka/api/FetchResponse.scala | 25 ++++++---- .../main/scala/kafka/network/RequestChannel.scala | 10 ++-- .../main/scala/kafka/network/SocketServer.scala | 20 +++++--- .../main/scala/kafka/producer/SyncProducer.scala | 4 +- core/src/main/scala/kafka/server/KafkaApis.scala | 2 +- 10 files changed, 89 insertions(+), 63 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index 0c13a7d..ca5210b 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -46,6 +46,7 @@ public class ByteBufferSend implements Send { return remaining <= 0; } + @Override public int size() { return this.size; } @@ -60,5 +61,4 @@ public class ByteBufferSend implements Send { remaining -= written; return (int) written; } - } diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index c122de2..44f669e 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -16,13 +16,14 @@ */ package org.apache.kafka.common.network; +import org.apache.kafka.common.KafkaException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.channels.GatheringByteChannel; +import java.util.Iterator; import java.util.List; -import java.util.ListIterator; /** * A set of composite sends, sent one after another @@ -31,21 +32,28 @@ import java.util.ListIterator; public class MultiSend implements Send { private static final Logger log = LoggerFactory.getLogger(MultiSend.class); - private static String dest; - private static List sends; - private static long expectedBytesToWrite; + private String dest; private long totalWritten = 0; - private ListIterator sendListIterator; + private List sends; + private Iterator sendsIterator; + private Send current; + private boolean doneSends = false; + private int size = 0; - //private int current; - //private var current = sends - //public static long totalWritten = 0; - - public MultiSend(String dest, List sends, long expectedBytesToWrite) { + public MultiSend(String dest, List sends) { this.dest = dest; this.sends = sends; - this.expectedBytesToWrite = expectedBytesToWrite; - sendListIterator = sends.listIterator(); + this.sendsIterator = sends.iterator(); + if (sendsIterator.hasNext()) + this.current = sendsIterator.next(); + + for (Send send: sends) + this.size += send.size(); + } + + @Override + public int size() { + return size; } @Override @@ -55,9 +63,9 @@ public class MultiSend implements Send { @Override public boolean completed() { - if (!sendListIterator.hasNext()) { - if (totalWritten != expectedBytesToWrite) - log.error("mismatch in sending bytes over socket; expected: " + expectedBytesToWrite + " actual: " + totalWritten); + if (doneSends) { + if (totalWritten != size) + log.error("mismatch in sending bytes over socket; expected: " + size + " actual: " + totalWritten); return true; } else { return false; @@ -66,24 +74,24 @@ public class MultiSend implements Send { @Override public int writeTo(GatheringByteChannel channel) throws IOException { - //TODO: Is this needed? expectIncomplete + if (completed()) + throw new KafkaException("This operation cannot be completed on a complete request."); + int totalWrittenPerCall = 0; boolean sendComplete = false; - do { - Send current = sendListIterator.next(); - int written = current.writeTo(channel); + long written = current.writeTo(channel); totalWritten += written; totalWrittenPerCall += written; sendComplete = current.completed(); - if (!sendComplete) { - // move iterator back so we will keep processing same send - sendListIterator.previous(); - } - + if (sendComplete) + if (sendsIterator.hasNext()) + current = sendsIterator.next(); + else + doneSends = true; } while (!completed() && sendComplete); if (log.isTraceEnabled()) { - log.trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + expectedBytesToWrite); + log.trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + size); } return totalWrittenPerCall; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index c3d9db0..5e3a4a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -23,8 +23,8 @@ import java.nio.channels.ScatteringByteChannel; */ public class NetworkReceive implements Receive { - private final static String UNKNOWN_SOURCE = ""; - private final static int UNLIMITED = -1; + public final static String UNKNOWN_SOURCE = ""; + public final static int UNLIMITED = -1; private final String source; private final ByteBuffer size; @@ -84,12 +84,12 @@ public class NetworkReceive implements Receive { read += bytesRead; if (!size.hasRemaining()) { size.rewind(); - int requestSize = size.getInt(); - if (requestSize < 0) - throw new IllegalStateException("Invalid request (size = " + requestSize + ")"); - if (maxSize != UNLIMITED && requestSize > maxSize) - throw new IllegalStateException("Invalid request (size = " + requestSize + " larger than " + maxSize + ")"); - this.buffer = ByteBuffer.allocate(requestSize); + int receiveSize = size.getInt(); + if (receiveSize < 0) + throw new IllegalStateException("Invalid request (size = " + receiveSize + ")"); + if (maxSize != UNLIMITED && receiveSize > maxSize) + throw new IllegalStateException("Invalid request (size = " + receiveSize + " larger than " + maxSize + ")"); + this.buffer = ByteBuffer.allocate(receiveSize); } } if (buffer != null) { diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index f79f12b..2ffaea5 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -107,13 +107,14 @@ public class Selector implements Selectable { this.disconnected = new ArrayList(); this.failedSends = new ArrayList(); this.sensors = new SelectorMetrics(metrics); - this.lruConnections = new LinkedHashMap(); + // initial capacity and load factor are default, we set them explicitly because we want to set accessOrder = true + this.lruConnections = new LinkedHashMap(16, .75F, true); currentTimeNanos = new SystemTime().nanoseconds(); nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; } public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(-1, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags); + this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags); } /** @@ -283,8 +284,7 @@ public class Selector implements Selectable { transmissions.send.writeTo(channel); if (transmissions.send.completed()) { this.completedSends.add(transmissions.send); - // TODO: Figure out the metrics here. Send doesn't have "size" in general and the server-side records sent messages, not sent bytes. - this.sensors.recordBytesSent(transmissions.id, 0); + this.sensors.recordBytesSent(transmissions.id, transmissions.send.size()); transmissions.clearSend(); key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE); } @@ -347,7 +347,7 @@ public class Selector implements Selectable { } @Override - public void mute(String id) { + public void mute(String id) { mute(this.keyForId(id)); } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index 75f5001..00e0a94 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -39,4 +39,9 @@ public interface Send { */ public int writeTo(GatheringByteChannel channel) throws IOException; + /** + * Size of the send + */ + public int size(); + } diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 6d2130e..1e4007c 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -23,6 +23,7 @@ import java.nio.channels.GatheringByteChannel import kafka.common.{TopicAndPartition, ErrorMapping} import kafka.message.{MessageSet, ByteBufferMessageSet} import kafka.api.ApiUtils._ +import org.apache.kafka.common.KafkaException import org.apache.kafka.common.network.Send import org.apache.kafka.common.network.MultiSend @@ -78,6 +79,8 @@ class PartitionDataSend(val partitionId: Int, } written } + + override def size = messageSize } object TopicData { @@ -105,7 +108,7 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti } class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { - private val size = topicData.sizeInBytes + override val size = topicData.sizeInBytes private var sent = 0 @@ -118,11 +121,13 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { buffer.putInt(topicData.partitionData.size) buffer.rewind() - val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(topicData.partitionData.toList - .map(d => new PartitionDataSend(d._1, d._2))), topicData.sizeInBytes - topicData.headerSize) + val sends = new MultiSend(dest, + JavaConversions.seqAsJavaList(topicData.partitionData.toList.map(d => new PartitionDataSend(d._1, d._2)))) + + override def writeTo(channel: GatheringByteChannel): Int = { + if (completed) + throw new KafkaException("This operation cannot be completed on a complete request.") - def writeTo(channel: GatheringByteChannel): Int = { - // TODO: expectIncomplete() var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) @@ -204,7 +209,7 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { - private val size = fetchResponse.sizeInBytes + override val size = fetchResponse.sizeInBytes private var sent = 0 @@ -223,10 +228,12 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(fetchResponse.dataGroupedByTopic.toList.map { case(topic, data) => new TopicDataSend(dest, TopicData(topic, data.map{case(topicAndPartition, message) => (topicAndPartition.partition, message)})) - }), fetchResponse.sizeInBytes - FetchResponse.headerSize) + })) + + override def writeTo(channel: GatheringByteChannel): Int = { + if (completed) + throw new KafkaException("This operation cannot be completed on a complete request.") - def writeTo(channel: GatheringByteChannel): Int = { - //TODO: expectIncomplete() var written = 0 if(buffer.hasRemaining) written += channel.write(buffer) diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 4f75cbf..916e440 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -44,7 +44,7 @@ object RequestChannel extends Logging { byteBuffer } - case class Request(processor: Int, connectionId: String, private var buffer: ByteBuffer, startTimeMs: Long, remoteAddress: SocketAddress = new InetSocketAddress(0), securityProtocol: SecurityProtocol) { + case class Request(processor: Int, connectionId: String, private var buffer: ByteBuffer, startTimeMs: Long, securityProtocol: SecurityProtocol) { @volatile var requestDequeueTimeMs = -1L @volatile var apiLocalCompleteTimeMs = -1L @volatile var responseCompleteTimeMs = -1L @@ -101,11 +101,11 @@ object RequestChannel extends Logging { m.totalTimeHist.update(totalTime) } if(requestLogger.isTraceEnabled) - requestLogger.trace("Completed request:%s from client %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" - .format(requestObj.describe(true), remoteAddress, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) + requestLogger.trace("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" + .format(requestObj.describe(true), connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) else if(requestLogger.isDebugEnabled) { - requestLogger.debug("Completed request:%s from client %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" - .format(requestObj.describe(false), remoteAddress, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) + requestLogger.debug("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" + .format(requestObj.describe(false), connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) } } } diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 3ff1dd1..690097d 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -102,7 +102,7 @@ class SocketServer(val brokerId: Int, */ // register the processor threads for notification of responses - requestChannel.addResponseListener((id:Int) => processors(id).wakeup()) + requestChannel.addResponseListener((id:Int) => processors(id).wakeup) // start accepting connections // right now we will use the same processors for all ports, since we didn't implement different protocols @@ -150,6 +150,8 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ private val shutdownLatch = new CountDownLatch(1) private val alive = new AtomicBoolean(true) + def wakeup + def shutdownHook /** @@ -306,9 +308,11 @@ private[kafka] class Acceptor(val host: String, /** * Wakeup the thread for selection. */ - def wakeup() = nioSelector.wakeup() + @Override + def wakeup = nioSelector.wakeup - def shutdownHook = wakeup() + @Override + def shutdownHook = wakeup } /** @@ -349,7 +353,7 @@ private[kafka] class Processor(val id: Int, // register any new responses for writing processNewResponses() val startSelectTime = new SystemTime().nanoseconds - val ready = selector.poll(300) + selector.poll(300) collection.JavaConversions.iterableAsScalaIterable(selector.completedReceives()).foreach( receive => { val req = RequestChannel.Request(processor = id, connectionId = receive.source(), buffer = receive.payload(), startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) requestChannel.sendRequest(req) @@ -397,7 +401,7 @@ private[kafka] class Processor(val id: Int, */ def accept(socketChannel: SocketChannel) { newConnections.add(socketChannel) - wakeup() + wakeup } /** @@ -406,7 +410,7 @@ private[kafka] class Processor(val id: Int, private def configureNewConnections() { while(!newConnections.isEmpty) { val channel = newConnections.poll() - info("Processor " + id + " listening to new connection from " + channel.socket.getRemoteSocketAddress) + debug("Processor " + id + " listening to new connection from " + channel.socket.getRemoteSocketAddress) val localHost = channel.socket().getLocalAddress.getHostAddress val localPort = channel.socket().getLocalPort val remoteHost = channel.socket().getInetAddress.getHostAddress @@ -426,8 +430,10 @@ private[kafka] class Processor(val id: Int, /** * Wakeup the thread for selection. */ - def wakeup() = selector.wakeup() + @Override + def wakeup = selector.wakeup + @Override def shutdownHook = wakeup } diff --git a/core/src/main/scala/kafka/producer/SyncProducer.scala b/core/src/main/scala/kafka/producer/SyncProducer.scala index 6e5f4d3..dcee501 100644 --- a/core/src/main/scala/kafka/producer/SyncProducer.scala +++ b/core/src/main/scala/kafka/producer/SyncProducer.scala @@ -105,14 +105,14 @@ class SyncProducer(val config: SyncProducerConfig) extends Logging { } } if(producerRequest.requiredAcks != 0) - ProducerResponse.readFrom(response.payload()) + ProducerResponse.readFrom(response.payload) else null } def send(request: TopicMetadataRequest): TopicMetadataResponse = { val response = doSend(request) - TopicMetadataResponse.readFrom(response.payload()) + TopicMetadataResponse.readFrom(response.payload) } def close() = { diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 047cd13..3fbd031 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -54,7 +54,7 @@ class KafkaApis(val requestChannel: RequestChannel, */ def handle(request: RequestChannel.Request) { try{ - trace("Handling request: " + request.requestObj + " from client: " + request.remoteAddress) + trace("Handling request: " + request.requestObj + " from connection: " + request.connectionId) request.requestId match { case RequestKeys.ProduceKey => handleProducerRequest(request) case RequestKeys.FetchKey => handleFetchRequest(request) -- 2.3.2 (Apple Git-55) From 3c3cc5d01261711f99a09b08ce1e4043f379ac3a Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Thu, 14 May 2015 14:28:48 +0300 Subject: [PATCH 11/24] fixed some issues with multisend --- .../java/org/apache/kafka/common/network/MultiSend.java | 17 ++++++++++------- core/src/main/scala/kafka/api/FetchResponse.scala | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index 44f669e..f7cfcfe 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -44,9 +44,7 @@ public class MultiSend implements Send { this.dest = dest; this.sends = sends; this.sendsIterator = sends.iterator(); - if (sendsIterator.hasNext()) - this.current = sendsIterator.next(); - + nextSendOrDone(); for (Send send: sends) this.size += send.size(); } @@ -85,14 +83,19 @@ public class MultiSend implements Send { totalWrittenPerCall += written; sendComplete = current.completed(); if (sendComplete) - if (sendsIterator.hasNext()) - current = sendsIterator.next(); - else - doneSends = true; + nextSendOrDone(); } while (!completed() && sendComplete); if (log.isTraceEnabled()) { log.trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + size); } return totalWrittenPerCall; } + + // update current if there's a next Send, mark sends as done if there isn't + private void nextSendOrDone() { + if (sendsIterator.hasNext()) + current = sendsIterator.next(); + else + doneSends = true; + } } \ No newline at end of file diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 1e4007c..9e54ff0 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -80,7 +80,7 @@ class PartitionDataSend(val partitionId: Int, written } - override def size = messageSize + override def size = buffer.capacity() + messageSize } object TopicData { -- 2.3.2 (Apple Git-55) From a6078914f7ac224b98f8c42d3dd0b9996a2caa0a Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Fri, 15 May 2015 00:11:56 +0300 Subject: [PATCH 12/24] fixed metric thingies --- core/src/main/scala/kafka/network/SocketServer.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 402df81..55fd3af 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -21,12 +21,11 @@ import java.util import java.util.concurrent._ import java.util.concurrent.atomic._ import java.net._ -import java.io._ import java.nio.channels._ +import com.yammer.metrics.core.Meter import kafka.cluster.EndPoint import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.network.NetworkReceive import org.apache.kafka.common.protocol.SecurityProtocol import org.apache.kafka.common.utils.{Time, SystemTime} @@ -35,7 +34,6 @@ import scala.collection._ import kafka.common.KafkaException import kafka.metrics.KafkaMetricsGroup import kafka.utils._ -import com.yammer.metrics.core.{Gauge, Meter} import org.apache.kafka.common.utils.Utils /** @@ -85,7 +83,6 @@ class SocketServer(val brokerId: Int, time, maxRequestSize, aggregateIdleMeter, - newMeter("IdlePercent", "percent", TimeUnit.NANOSECONDS, Map("networkProcessor" -> i.toString)), numProcessorThreads, requestChannel, quotas, @@ -318,7 +315,6 @@ private[kafka] class Processor(val id: Int, val time: Time, val maxRequestSize: Int, val aggregateIdleMeter: Meter, - val idleMeter: Meter, val totalProcessorThreads: Int, val requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, @@ -327,7 +323,7 @@ private[kafka] class Processor(val id: Int, val metrics: Metrics) extends AbstractServerThread(connectionQuotas) { private val newConnections = new ConcurrentLinkedQueue[SocketChannel]() - private val lruConnections = new util.LinkedHashMap[SelectionKey, Long] + private val inflightResponses = mutable.Map[String, RequestChannel.Response]() private var metricTags = new util.HashMap[String, String]() metricTags.put("processor-id", id.toString) @@ -368,6 +364,12 @@ private[kafka] class Processor(val id: Int, }) } + collection.JavaConversions.iterableAsScalaIterable(selector.completedSends()).foreach( send => { + val resp = inflightResponses.remove(send.destination()).get + resp.request.updateRequestMetrics() + selector.unmute(send.destination()) + }) + debug("Closing selector - processor " + id) closeAll() shutdownComplete() @@ -388,6 +390,7 @@ private[kafka] class Processor(val id: Int, case RequestChannel.SendAction => { trace("Socket server received response to send, registering for write, sending data and registering for read: " + curr) selector.send(curr.responseSend) + inflightResponses += (curr.request.connectionId -> curr) selector.unmute(curr.responseSend.destination()) } case RequestChannel.CloseConnectionAction => { -- 2.3.2 (Apple Git-55) From d3ed5d62d333ada8b25df2ea7b67b1fe29d00c36 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Fri, 15 May 2015 00:39:55 +0300 Subject: [PATCH 13/24] fixed response order bug --- core/src/main/scala/kafka/network/SocketServer.scala | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 55fd3af..2934a46 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -362,13 +362,15 @@ private[kafka] class Processor(val id: Int, } selector.mute(receive.source) }) + + collection.JavaConversions.iterableAsScalaIterable(selector.completedSends()).foreach( send => { + val resp = inflightResponses.remove(send.destination()).get + resp.request.updateRequestMetrics() + selector.unmute(send.destination()) + }) } - collection.JavaConversions.iterableAsScalaIterable(selector.completedSends()).foreach( send => { - val resp = inflightResponses.remove(send.destination()).get - resp.request.updateRequestMetrics() - selector.unmute(send.destination()) - }) + debug("Closing selector - processor " + id) closeAll() @@ -391,7 +393,6 @@ private[kafka] class Processor(val id: Int, trace("Socket server received response to send, registering for write, sending data and registering for read: " + curr) selector.send(curr.responseSend) inflightResponses += (curr.request.connectionId -> curr) - selector.unmute(curr.responseSend.destination()) } case RequestChannel.CloseConnectionAction => { curr.request.updateRequestMetrics -- 2.3.2 (Apple Git-55) From 40870238c8302564e30e8302b47d595ee4b15112 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Fri, 15 May 2015 03:00:16 +0300 Subject: [PATCH 14/24] error handling for illegal selector state and fix metrics bug --- .../java/org/apache/kafka/common/network/Selector.java | 5 ----- core/src/main/scala/kafka/network/RequestChannel.scala | 17 ++++++++++++++--- core/src/main/scala/kafka/network/SocketServer.scala | 10 +++++++++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 779a3ee..6744188 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -305,11 +305,6 @@ public class Selector implements Selectable { log.warn("Error in I/O with connection to {}", desc, e); close(transmissions.id); this.disconnected.add(transmissions.id); - } catch (IllegalStateException e) { - String desc = socketDescription(channel); - log.warn("Closing socket connection to {} due to illegal state {}", desc, e); - close(transmissions.id); - this.disconnected.add(transmissions.id); } } } diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 916e440..263e23e 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -100,12 +100,23 @@ object RequestChannel extends Logging { m.responseSendTimeHist.update(responseSendTime) m.totalTimeHist.update(totalTime) } - if(requestLogger.isTraceEnabled) + if(requestLogger.isTraceEnabled) { + val requestDesc = + if (requestObj != null) + requestObj.describe(true) + else + header.toString + " -- " + body.toString requestLogger.trace("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" - .format(requestObj.describe(true), connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) + .format(requestDesc, connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) + } else if(requestLogger.isDebugEnabled) { + val requestDesc = + if (requestObj != null) + requestObj.describe(false) + else + header.toString + " -- " + body.toString requestLogger.debug("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" - .format(requestObj.describe(false), connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) + .format(requestDesc, connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) } } } diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 2934a46..43de522 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -345,7 +345,15 @@ private[kafka] class Processor(val id: Int, // register any new responses for writing processNewResponses() - selector.poll(300) + try { + selector.poll(300) + } catch { + case e: IllegalStateException => { + error("Closing processor %s due to illegal state".format(id)) + swallow(closeAll()) + shutdownComplete() + } + } collection.JavaConversions.iterableAsScalaIterable(selector.completedReceives).foreach( receive => { val req = RequestChannel.Request(processor = id, connectionId = receive.source, buffer = receive.payload, startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) -- 2.3.2 (Apple Git-55) From 7123119acb5e34d5ea0111bee7300efa8fbe945e Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Fri, 15 May 2015 10:26:34 +0300 Subject: [PATCH 15/24] optimized selection key lookup with identity hash --- .../src/main/java/org/apache/kafka/common/network/Selector.java | 9 ++++++--- core/src/main/scala/kafka/network/SocketServer.scala | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 6744188..9eafa1a 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -90,7 +90,7 @@ public class Selector implements Selectable { /** * Create a new nioSelector */ - public Selector(int maxReceiveSize, long connectionMaxIdleMs, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags, boolean metricsPerConnection) { + public Selector(int maxReceiveSize, long connectionMaxIdleMs, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags, boolean metricsPerConnection, boolean useIdentityMapForKeys) { try { this.nioSelector = java.nio.channels.Selector.open(); } catch (IOException e) { @@ -101,7 +101,10 @@ public class Selector implements Selectable { this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; - this.keys = new HashMap(); + if (useIdentityMapForKeys) + this.keys = new IdentityHashMap(); + else + this.keys = new HashMap(); this.completedSends = new ArrayList(); this.completedReceives = new ArrayList(); this.connected = new ArrayList(); @@ -116,7 +119,7 @@ public class Selector implements Selectable { } public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true); + this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true, false); } /** diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 43de522..c90658e 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -328,13 +328,14 @@ private[kafka] class Processor(val id: Int, private var metricTags = new util.HashMap[String, String]() metricTags.put("processor-id", id.toString) private val selector = new org.apache.kafka.common.network.Selector( - maxRequestSize, + maxRequestSize, connectionsMaxIdleMs, metrics, time, "SocketServer", metricTags, - false + false, + true ) override def run() { -- 2.3.2 (Apple Git-55) From 51c2f631c994763f447b11249c32ee661c75caea Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Mon, 18 May 2015 10:02:46 +0300 Subject: [PATCH 16/24] fix accidental change --- core/src/main/scala/kafka/network/SocketServer.scala | 2 +- core/src/test/resources/log4j.properties | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index c90658e..a273c45 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -328,7 +328,7 @@ private[kafka] class Processor(val id: Int, private var metricTags = new util.HashMap[String, String]() metricTags.put("processor-id", id.toString) private val selector = new org.apache.kafka.common.network.Selector( - maxRequestSize, + maxRequestSize, connectionsMaxIdleMs, metrics, time, diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties index 9973dad..1b7d5d8 100644 --- a/core/src/test/resources/log4j.properties +++ b/core/src/test/resources/log4j.properties @@ -12,14 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -log4j.rootLogger=TRACE, stdout +log4j.rootLogger=OFF, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n -log4j.logger.kafka=TRACE -log4j.logger.org.apache.kafka=TRACE +log4j.logger.kafka=ERROR +log4j.logger.org.apache.kafka=ERROR # zkclient can be verbose, during debugging it is common to adjust is separately log4j.logger.org.I0Itec.zkclient.ZkClient=WARN -- 2.3.2 (Apple Git-55) From 74dcee05f540c602708e40ae424e19bc74413372 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Mon, 18 May 2015 17:45:22 +0300 Subject: [PATCH 17/24] addressing Jun's comments --- .../org/apache/kafka/clients/NetworkClient.java | 2 +- .../kafka/common/network/ByteBufferSend.java | 2 +- .../org/apache/kafka/common/network/MultiSend.java | 5 +-- .../apache/kafka/common/network/Selectable.java | 2 +- .../org/apache/kafka/common/network/Selector.java | 37 ++++++++++------ .../java/org/apache/kafka/common/network/Send.java | 2 +- .../apache/kafka/common/network/SelectorTest.java | 8 ++-- .../java/org/apache/kafka/test/MockSelector.java | 8 ++-- core/src/main/scala/kafka/api/FetchResponse.scala | 20 ++++----- .../main/scala/kafka/network/SocketServer.scala | 51 ++++++++-------------- 10 files changed, 67 insertions(+), 70 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 48fe796..7569b15 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -371,7 +371,7 @@ public class NetworkClient implements KafkaClient { * @param now The current time */ private void handleCompletedReceives(List responses, long now) { - for (NetworkReceive receive : this.selector.completedReceives()) { + for (NetworkReceive receive : this.selector.completedReceives().values()) { String source = receive.source(); ClientRequest req = inFlightRequests.completeNext(source); ResponseHeader header = ResponseHeader.parse(receive.payload()); diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index ca5210b..5c32018 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -52,7 +52,7 @@ public class ByteBufferSend implements Send { } @Override - public int writeTo(GatheringByteChannel channel) throws IOException { + public long writeTo(GatheringByteChannel channel) throws IOException { long written = channel.write(buffers); if (written < 0) throw new EOFException("This shouldn't happen."); diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index f7cfcfe..88f283f 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -71,7 +71,7 @@ public class MultiSend implements Send { } @Override - public int writeTo(GatheringByteChannel channel) throws IOException { + public long writeTo(GatheringByteChannel channel) throws IOException { if (completed()) throw new KafkaException("This operation cannot be completed on a complete request."); @@ -85,9 +85,8 @@ public class MultiSend implements Send { if (sendComplete) nextSendOrDone(); } while (!completed() && sendComplete); - if (log.isTraceEnabled()) { + if (log.isTraceEnabled()) log.trace("Bytes written as part of multisend call : " + totalWrittenPerCall + "Total bytes written so far : " + totalWritten + "Expected bytes to write : " + size); - } return totalWrittenPerCall; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java index 618a0fa..aa0d5f3 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java @@ -67,7 +67,7 @@ public interface Selectable { /** * The list of receives that completed on the last {@link #poll(long, List) poll()} call. */ - public List completedReceives(); + public java.util.Map completedReceives(); /** * The list of connections that finished disconnecting on the last {@link #poll(long, List) poll()} diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 9eafa1a..414658b 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -71,7 +71,7 @@ public class Selector implements Selectable { private final java.nio.channels.Selector nioSelector; private final Map keys; private final List completedSends; - private final List completedReceives; + private final Map completedReceives; private final List disconnected; private final List connected; private final List failedSends; @@ -90,7 +90,7 @@ public class Selector implements Selectable { /** * Create a new nioSelector */ - public Selector(int maxReceiveSize, long connectionMaxIdleMs, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags, boolean metricsPerConnection, boolean useIdentityMapForKeys) { + public Selector(int maxReceiveSize, long connectionMaxIdleMs, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags, boolean metricsPerConnection) { try { this.nioSelector = java.nio.channels.Selector.open(); } catch (IOException e) { @@ -101,12 +101,9 @@ public class Selector implements Selectable { this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; - if (useIdentityMapForKeys) - this.keys = new IdentityHashMap(); - else - this.keys = new HashMap(); + this.keys = new HashMap(); this.completedSends = new ArrayList(); - this.completedReceives = new ArrayList(); + this.completedReceives = new HashMap(); this.connected = new ArrayList(); this.disconnected = new ArrayList(); this.failedSends = new ArrayList(); @@ -119,7 +116,7 @@ public class Selector implements Selectable { } public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true, false); + this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true); } /** @@ -275,10 +272,15 @@ public class Selector implements Selectable { if (key.isReadable()) { if (!transmissions.hasReceive()) transmissions.receive = new NetworkReceive(maxReceiveSize, transmissions.id); - transmissions.receive.readFrom(channel); + try { + transmissions.receive.readFrom(channel); + } catch (IllegalStateException e) { + log.error("Illegal state exception when receiving data from " + getConnectionId(key), e); + throw e; + } if (transmissions.receive.complete()) { transmissions.receive.payload().rewind(); - this.completedReceives.add(transmissions.receive); + this.completedReceives.put(getConnectionId(key), transmissions.receive); this.sensors.recordBytesReceived(transmissions.id, transmissions.receive.payload().limit()); transmissions.clearReceive(); } @@ -315,7 +317,16 @@ public class Selector implements Selectable { this.sensors.ioTime.record(endIo - endSelect, time.milliseconds()); maybeCloseOldestConnection(); } - + + private String getConnectionId(SelectionKey key) { + SocketChannel channel = (SocketChannel) key.channel(); + String localHost = channel.socket().getLocalAddress().getHostAddress(); + int localPort = channel.socket().getLocalPort(); + String remoteHost = channel.socket().getInetAddress().getHostAddress(); + int remotePort = channel.socket().getPort(); + return localHost + ":" + localPort + "-" + remoteHost + ":" + remotePort; + } + private String socketDescription(SocketChannel channel) { Socket socket = channel.socket(); if (socket == null) @@ -332,7 +343,7 @@ public class Selector implements Selectable { } @Override - public List completedReceives() { + public Map completedReceives() { return this.completedReceives; } @@ -603,7 +614,7 @@ public class Selector implements Selectable { } } - public void recordBytesSent(String node, int bytes) { + public void recordBytesSent(String node, long bytes) { long now = time.milliseconds(); this.bytesSent.record(bytes, now); if (!node.isEmpty()) { diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index 00e0a94..a0c07ad 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -37,7 +37,7 @@ public interface Send { * @return The number of bytes written * @throws IOException If the write fails */ - public int writeTo(GatheringByteChannel channel) throws IOException; + public long writeTo(GatheringByteChannel channel) throws IOException; /** * Size of the send diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index d23b4b6..02d0c76 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -166,7 +166,7 @@ public class SelectorTest { assertEquals("No disconnects should have occurred.", 0, selector.disconnected().size()); // handle any responses we may have gotten - for (NetworkReceive receive : selector.completedReceives()) { + for (NetworkReceive receive : selector.completedReceives().values()) { String[] pieces = asString(receive).split("-"); assertEquals("Should be in the form 'conn-counter'", 2, pieces.length); assertEquals("Check the source", receive.source(), pieces[0]); @@ -234,14 +234,14 @@ public class SelectorTest { while (selector.completedReceives().isEmpty()) selector.poll(5); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should not be from the muted node", "0", selector.completedReceives().get(0).source()); + assertEquals("The response should not be from the muted node", "0", selector.completedReceives().values().iterator().next().source()); selector.unmute("1"); do { selector.poll(5); } while (selector.completedReceives().isEmpty()); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should be from the previously muted node", "1", selector.completedReceives().get(0).source()); + assertEquals("The response should be from the previously muted node", "1", selector.completedReceives().values().iterator().next().source()); } private String blockingRequest(String node, String s) throws IOException { @@ -249,7 +249,7 @@ public class SelectorTest { selector.poll(1000L); while (true) { selector.poll(1000L); - for (NetworkReceive receive : selector.completedReceives()) + for (NetworkReceive receive : selector.completedReceives().values()) if (receive.source() == node) return asString(receive); } diff --git a/clients/src/test/java/org/apache/kafka/test/MockSelector.java b/clients/src/test/java/org/apache/kafka/test/MockSelector.java index 51eb9d1..af16129 100644 --- a/clients/src/test/java/org/apache/kafka/test/MockSelector.java +++ b/clients/src/test/java/org/apache/kafka/test/MockSelector.java @@ -15,7 +15,9 @@ package org.apache.kafka.test; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; @@ -31,7 +33,7 @@ public class MockSelector implements Selectable { private final Time time; private final List initiatedSends = new ArrayList(); private final List completedSends = new ArrayList(); - private final List completedReceives = new ArrayList(); + private final Map completedReceives = new HashMap(); private final List disconnected = new ArrayList(); private final List connected = new ArrayList(); @@ -86,12 +88,12 @@ public class MockSelector implements Selectable { } @Override - public List completedReceives() { + public java.util.Map completedReceives() { return completedReceives; } public void completeReceive(NetworkReceive receive) { - this.completedReceives.add(receive); + this.completedReceives.put("mock connection id", receive); } @Override diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 9e54ff0..ca6b571 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -68,8 +68,8 @@ class PartitionDataSend(val partitionId: Int, override def destination: String = "" - override def writeTo(channel: GatheringByteChannel): Int = { - var written = 0 + override def writeTo(channel: GatheringByteChannel): Long = { + var written = 0L if(buffer.hasRemaining) written += channel.write(buffer) if(!buffer.hasRemaining && messagesSentSize < messageSize) { @@ -110,7 +110,7 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { override val size = topicData.sizeInBytes - private var sent = 0 + private var sent = 0L override def completed: Boolean = sent >= size @@ -121,14 +121,14 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { buffer.putInt(topicData.partitionData.size) buffer.rewind() - val sends = new MultiSend(dest, + private val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(topicData.partitionData.toList.map(d => new PartitionDataSend(d._1, d._2)))) - override def writeTo(channel: GatheringByteChannel): Int = { + override def writeTo(channel: GatheringByteChannel): Long = { if (completed) throw new KafkaException("This operation cannot be completed on a complete request.") - var written = 0 + var written = 0L if(buffer.hasRemaining) written += channel.write(buffer) if(!buffer.hasRemaining && !sends.completed) { @@ -211,7 +211,7 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { override val size = fetchResponse.sizeInBytes - private var sent = 0 + private var sent = 0L private val sendSize = 4 /* for size */ + size @@ -225,16 +225,16 @@ class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) exte buffer.putInt(fetchResponse.dataGroupedByTopic.size) // topic count buffer.rewind() - val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(fetchResponse.dataGroupedByTopic.toList.map { + private val sends = new MultiSend(dest, JavaConversions.seqAsJavaList(fetchResponse.dataGroupedByTopic.toList.map { case(topic, data) => new TopicDataSend(dest, TopicData(topic, data.map{case(topicAndPartition, message) => (topicAndPartition.partition, message)})) })) - override def writeTo(channel: GatheringByteChannel): Int = { + override def writeTo(channel: GatheringByteChannel): Long = { if (completed) throw new KafkaException("This operation cannot be completed on a complete request.") - var written = 0 + var written = 0L if(buffer.hasRemaining) written += channel.write(buffer) if(!buffer.hasRemaining && !sends.completed) { diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index a273c45..25cfcd0 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -59,10 +59,6 @@ class SocketServer(val brokerId: Int, private[network] var acceptors = mutable.Map[EndPoint,Acceptor]() val requestChannel = new RequestChannel(numProcessorThreads, maxQueuedRequests) - /* a meter to track the average free capacity of the network processors */ - private val aggregateIdleMeter = newMeter("NetworkProcessorAvgIdlePercent", "percent", TimeUnit.NANOSECONDS) - - /* I'm pushing the mapping of port-to-protocol to the processor level, so the processor can put the correct protocol in the request channel. we'll probably have a more elegant way of doing this once we patch the request channel @@ -82,7 +78,6 @@ class SocketServer(val brokerId: Int, processors(i) = new Processor(i, time, maxRequestSize, - aggregateIdleMeter, numProcessorThreads, requestChannel, quotas, @@ -94,7 +89,7 @@ class SocketServer(val brokerId: Int, } // register the processor threads for notification of responses - requestChannel.addResponseListener((id:Int) => processors(id).wakeup) + requestChannel.addResponseListener((id:Int) => processors(id).wakeup()) // start accepting connections // right now we will use the same processors for all ports, since we didn't implement different protocols @@ -142,17 +137,15 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ private val shutdownLatch = new CountDownLatch(1) private val alive = new AtomicBoolean(true) - def wakeup - - def shutdownHook + def wakeup() /** * Initiates a graceful shutdown by signaling to stop and waiting for the shutdown to complete */ def shutdown(): Unit = { alive.set(false) - shutdownHook - shutdownLatch.await + wakeup() + shutdownLatch.await() } /** @@ -164,13 +157,13 @@ private[kafka] abstract class AbstractServerThread(connectionQuotas: ConnectionQ * Record that the thread startup is complete */ protected def startupComplete() = { - startupLatch.countDown + startupLatch.countDown() } /** * Record that the thread shutdown is complete */ - protected def shutdownComplete() = shutdownLatch.countDown + protected def shutdownComplete() = shutdownLatch.countDown() /** * Is the server still running? @@ -301,10 +294,8 @@ private[kafka] class Acceptor(val host: String, * Wakeup the thread for selection. */ @Override - def wakeup = nioSelector.wakeup + def wakeup = nioSelector.wakeup() - @Override - def shutdownHook = wakeup } /** @@ -314,7 +305,6 @@ private[kafka] class Acceptor(val host: String, private[kafka] class Processor(val id: Int, val time: Time, val maxRequestSize: Int, - val aggregateIdleMeter: Meter, val totalProcessorThreads: Int, val requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, @@ -334,9 +324,7 @@ private[kafka] class Processor(val id: Int, time, "SocketServer", metricTags, - false, - true - ) + false) override def run() { startupComplete() @@ -353,24 +341,24 @@ private[kafka] class Processor(val id: Int, error("Closing processor %s due to illegal state".format(id)) swallow(closeAll()) shutdownComplete() + throw e } } - collection.JavaConversions.iterableAsScalaIterable(selector.completedReceives).foreach( receive => { - val req = RequestChannel.Request(processor = id, connectionId = receive.source, buffer = receive.payload, startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) - + collection.JavaConversions.mapAsScalaMap(selector.completedReceives).foreach{ case (connectionId, receive) => { try { + val req = RequestChannel.Request(processor = id, connectionId = receive.source, buffer = receive.payload, startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) requestChannel.sendRequest(req) } catch { case e: InvalidRequestException => { - info("Closing socket connection to %s due to invalid request: %s".format(receive.source, e.getMessage)) - selector.close(receive.source) + info("Closing socket connection to %s due to invalid request: %s".format(connectionId, e.getMessage)) + selector.close(connectionId) } case e: Throwable => { - error("Closing socket for " + receive.source + " because of error", e) - selector.close(receive.source) + error("Closing socket for " + connectionId + " because of error", e) + selector.close(connectionId) } } selector.mute(receive.source) - }) + }} collection.JavaConversions.iterableAsScalaIterable(selector.completedSends()).foreach( send => { val resp = inflightResponses.remove(send.destination()).get @@ -422,7 +410,7 @@ private[kafka] class Processor(val id: Int, */ def accept(socketChannel: SocketChannel) { newConnections.add(socketChannel) - wakeup + wakeup() } /** @@ -452,10 +440,7 @@ private[kafka] class Processor(val id: Int, * Wakeup the thread for selection. */ @Override - def wakeup = selector.wakeup - - @Override - def shutdownHook = wakeup + def wakeup = selector.wakeup() } -- 2.3.2 (Apple Git-55) From 016669123a370b561b5ac78f8f1cf7bdd958e7d1 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Mon, 18 May 2015 18:55:33 +0300 Subject: [PATCH 18/24] removed connection-aging for clients --- .../kafka/clients/consumer/ConsumerConfig.java | 11 +-------- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../kafka/clients/producer/KafkaProducer.java | 2 +- .../kafka/clients/producer/ProducerConfig.java | 7 +----- .../org/apache/kafka/common/network/Selector.java | 28 +++++++++++++++------- .../apache/kafka/common/network/SelectorTest.java | 2 +- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index 55837cc..bdff518 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -156,10 +156,6 @@ public class ConsumerConfig extends AbstractConfig { public static final String VALUE_DESERIALIZER_CLASS_CONFIG = "value.deserializer"; private static final String VALUE_DESERIALIZER_CLASS_DOC = "Deserializer class for value that implements the Deserializer interface."; - /** connections.max.idle.ms */ - public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; - public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; - static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, @@ -281,12 +277,7 @@ public class ConsumerConfig extends AbstractConfig { .define(VALUE_DESERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, - VALUE_DESERIALIZER_CLASS_DOC) - .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, - Type.LONG, - 10 * 60 * 1000, - Importance.MEDIUM, - CONNECTIONS_MAX_IDLE_MS_DOC); + VALUE_DESERIALIZER_CLASS_DOC); } public static Map addDeserializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index d1d1ec1..79bce3d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -473,7 +473,7 @@ public class KafkaConsumer implements Consumer { Map metricsTags = new LinkedHashMap(); metricsTags.put("client-id", clientId); this.client = new NetworkClient( - new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, metricsTags), + new Selector(metrics, time, metricGrpPrefix, metricsTags), this.metadata, clientId, 100, // a fixed large enough value will suffice diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index 7891943..e1ee6b2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -229,7 +229,7 @@ public class KafkaProducer implements Producer { this.metadata.update(Cluster.bootstrap(addresses), time.milliseconds()); NetworkClient client = new NetworkClient( - new Selector(config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), this.metrics, time, "producer", metricTags), + new Selector(this.metrics, time, "producer", metricTags), this.metadata, clientId, config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index 7145fb0..187d000 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -169,10 +169,6 @@ public class ProducerConfig extends AbstractConfig { public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that implements the Serializer interface."; - /** connections.max.idle.ms */ - public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; - public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; - static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOSTRAP_SERVERS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, 32 * 1024 * 1024L, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) @@ -221,8 +217,7 @@ public class ProducerConfig extends AbstractConfig { Importance.LOW, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DOC) .define(KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) - .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, 10 * 60 * 1000, Importance.MEDIUM, CONNECTIONS_MAX_IDLE_MS_DOC); + .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC); } public static Map addSerializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 414658b..b197931 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -67,6 +67,7 @@ import org.slf4j.LoggerFactory; public class Selector implements Selectable { private static final Logger log = LoggerFactory.getLogger(Selector.class); + private static final long UNLIMITED = -1; private final java.nio.channels.Selector nioSelector; private final Map keys; @@ -97,7 +98,7 @@ public class Selector implements Selectable { throw new KafkaException(e); } this.maxReceiveSize = maxReceiveSize; - this.connectionsMaxIdleNanos = connectionMaxIdleMs * 1000 * 1000; + this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; @@ -108,15 +109,24 @@ public class Selector implements Selectable { this.disconnected = new ArrayList(); this.failedSends = new ArrayList(); this.sensors = new SelectorMetrics(metrics); - // initial capacity and load factor are default, we set them explicitly because we want to set accessOrder = true - this.lruConnections = new LinkedHashMap(16, .75F, true); + + // we only maintain LRU if we plan on aging out idle connections + if (connectionMaxIdleMs != UNLIMITED) { + this.connectionsMaxIdleNanos = connectionMaxIdleMs * 1000 * 1000; + // initial capacity and load factor are default, we set them explicitly because we want to set accessOrder = true + this.lruConnections = new LinkedHashMap(16, .75F, true); + } else { + this.connectionsMaxIdleNanos = connectionMaxIdleMs; + this.lruConnections = null; + } + currentTimeNanos = new SystemTime().nanoseconds(); nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; this.metricsPerConnection = metricsPerConnection; } - public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true); + public Selector(Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { + this(NetworkReceive.UNLIMITED, UNLIMITED, metrics, time, metricGrpPrefix, metricTags, true); } /** @@ -258,7 +268,8 @@ public class Selector implements Selectable { // register all per-broker metrics at once sensors.maybeRegisterNodeMetrics(transmissions.id); - lruConnections.put(transmissions.id, currentTimeNanos); + if (connectionsMaxIdleNanos != UNLIMITED) + lruConnections.put(transmissions.id, currentTimeNanos); try { /* complete any connections that have finished their handshake */ if (key.isConnectable()) { @@ -388,7 +399,7 @@ public class Selector implements Selectable { } private void maybeCloseOldestConnection() { - if (currentTimeNanos > nextIdleCloseCheckTime) { + if (connectionsMaxIdleNanos != UNLIMITED && currentTimeNanos > nextIdleCloseCheckTime) { if (lruConnections.isEmpty()) { nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; } else { @@ -439,7 +450,8 @@ public class Selector implements Selectable { */ public void close(String id) { SelectionKey key = keyForId(id); - lruConnections.remove(id); + if (connectionsMaxIdleNanos != UNLIMITED) + lruConnections.remove(id); SocketChannel channel = channel(key); Transmissions trans = transmissions(key); if (trans != null) { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 02d0c76..4e2c883 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -46,7 +46,7 @@ public class SelectorTest { public void setup() throws Exception { this.server = new EchoServer(); this.server.start(); - this.selector = new Selector(5000, new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); + this.selector = new Selector(new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); } @After -- 2.3.2 (Apple Git-55) From 46c4ed48fea2832288ebf41889946fad9c3472d5 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Tue, 19 May 2015 11:13:51 +0300 Subject: [PATCH 19/24] fix issues with exception handling and other cleanup --- .../org/apache/kafka/clients/NetworkClient.java | 2 +- .../kafka/common/network/ByteBufferSend.java | 6 +-- .../common/network/InvalidReceiveException.java | 30 +++++++++++++++ .../kafka/common/network/NetworkReceive.java | 4 +- .../apache/kafka/common/network/Selectable.java | 2 +- .../org/apache/kafka/common/network/Selector.java | 43 +++++++++++----------- .../apache/kafka/common/network/SelectorTest.java | 8 ++-- .../java/org/apache/kafka/test/MockSelector.java | 8 ++-- core/src/main/scala/kafka/api/RequestKeys.scala | 4 +- .../main/scala/kafka/network/BlockingChannel.scala | 5 ++- core/src/main/scala/kafka/network/Handler.scala | 2 +- .../main/scala/kafka/network/RequestChannel.scala | 8 ++-- .../main/scala/kafka/network/SocketServer.scala | 35 +++++++++--------- .../unit/kafka/network/SocketServerTest.scala | 8 +++- 14 files changed, 99 insertions(+), 66 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/common/network/InvalidReceiveException.java diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 7569b15..48fe796 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -371,7 +371,7 @@ public class NetworkClient implements KafkaClient { * @param now The current time */ private void handleCompletedReceives(List responses, long now) { - for (NetworkReceive receive : this.selector.completedReceives().values()) { + for (NetworkReceive receive : this.selector.completedReceives()) { String source = receive.source(); ClientRequest req = inFlightRequests.completeNext(source); ResponseHeader header = ResponseHeader.parse(receive.payload()); diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index 5c32018..3cdc07f 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -55,10 +55,8 @@ public class ByteBufferSend implements Send { public long writeTo(GatheringByteChannel channel) throws IOException { long written = channel.write(buffers); if (written < 0) - throw new EOFException("This shouldn't happen."); - if (written > Integer.MAX_VALUE) - throw new IOException("Wrote more bytes than " + size + ". This shouldn't happen."); + throw new EOFException("Wrote negative bytes to channel. This shouldn't happen."); remaining -= written; - return (int) written; + return written; } } diff --git a/clients/src/main/java/org/apache/kafka/common/network/InvalidReceiveException.java b/clients/src/main/java/org/apache/kafka/common/network/InvalidReceiveException.java new file mode 100644 index 0000000..a5bdd62 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/network/InvalidReceiveException.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.network; + +import org.apache.kafka.common.KafkaException; + +public class InvalidReceiveException extends KafkaException { + + public InvalidReceiveException(String message) { + super(message); + } + + public InvalidReceiveException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index 5e3a4a0..3ca0098 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/clients/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -86,9 +86,9 @@ public class NetworkReceive implements Receive { size.rewind(); int receiveSize = size.getInt(); if (receiveSize < 0) - throw new IllegalStateException("Invalid request (size = " + receiveSize + ")"); + throw new InvalidReceiveException("Invalid receive (size = " + receiveSize + ")"); if (maxSize != UNLIMITED && receiveSize > maxSize) - throw new IllegalStateException("Invalid request (size = " + receiveSize + " larger than " + maxSize + ")"); + throw new InvalidReceiveException("Invalid receive (size = " + receiveSize + " larger than " + maxSize + ")"); this.buffer = ByteBuffer.allocate(receiveSize); } } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java index aa0d5f3..618a0fa 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selectable.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selectable.java @@ -67,7 +67,7 @@ public interface Selectable { /** * The list of receives that completed on the last {@link #poll(long, List) poll()} call. */ - public java.util.Map completedReceives(); + public List completedReceives(); /** * The list of connections that finished disconnecting on the last {@link #poll(long, List) poll()} diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index b197931..6962639 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -72,7 +72,7 @@ public class Selector implements Selectable { private final java.nio.channels.Selector nioSelector; private final Map keys; private final List completedSends; - private final Map completedReceives; + private final List completedReceives; private final List disconnected; private final List connected; private final List failedSends; @@ -104,7 +104,7 @@ public class Selector implements Selectable { this.metricTags = metricTags; this.keys = new HashMap(); this.completedSends = new ArrayList(); - this.completedReceives = new HashMap(); + this.completedReceives = new ArrayList(); this.connected = new ArrayList(); this.disconnected = new ArrayList(); this.failedSends = new ArrayList(); @@ -266,8 +266,8 @@ public class Selector implements Selectable { Transmissions transmissions = transmissions(key); SocketChannel channel = channel(key); - // register all per-broker metrics at once - sensors.maybeRegisterNodeMetrics(transmissions.id); + // register all per-connection metrics at once + sensors.maybeRegisterConnectionMetrics(transmissions.id); if (connectionsMaxIdleNanos != UNLIMITED) lruConnections.put(transmissions.id, currentTimeNanos); try { @@ -285,13 +285,14 @@ public class Selector implements Selectable { transmissions.receive = new NetworkReceive(maxReceiveSize, transmissions.id); try { transmissions.receive.readFrom(channel); - } catch (IllegalStateException e) { - log.error("Illegal state exception when receiving data from " + getConnectionId(key), e); + } catch (InvalidReceiveException e) { + log.error("Illegal state exception when receiving data from " + getConnectionId(key) + " closing connection", e); + close(transmissions.id); throw e; } if (transmissions.receive.complete()) { transmissions.receive.payload().rewind(); - this.completedReceives.put(getConnectionId(key), transmissions.receive); + this.completedReceives.add(transmissions.receive); this.sensors.recordBytesReceived(transmissions.id, transmissions.receive.payload().limit()); transmissions.clearReceive(); } @@ -354,7 +355,7 @@ public class Selector implements Selectable { } @Override - public Map completedReceives() { + public List completedReceives() { return this.completedReceives; } @@ -587,17 +588,17 @@ public class Selector implements Selectable { }); } - public void maybeRegisterNodeMetrics(String node) { - if (!node.isEmpty() && metricsPerConnection) { - // if one sensor of the metrics has been registered for the node, + public void maybeRegisterConnectionMetrics(String connectionId) { + if (!connectionId.isEmpty() && metricsPerConnection) { + // if one sensor of the metrics has been registered for the connection, // then all other sensors should have been registered; and vice versa - String nodeRequestName = "node-" + node + ".bytes-sent"; + String nodeRequestName = "node-" + connectionId + ".bytes-sent"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest == null) { String metricGrpName = metricGrpPrefix + "-node-metrics"; Map tags = new LinkedHashMap(metricTags); - tags.put("node-id", "node-" + node); + tags.put("node-id", "node-" + connectionId); nodeRequest = this.metrics.sensor(nodeRequestName); MetricName metricName = new MetricName("outgoing-byte-rate", metricGrpName, tags); @@ -609,14 +610,14 @@ public class Selector implements Selectable { metricName = new MetricName("request-size-max", metricGrpName, "The maximum size of any request sent in the window.", tags); nodeRequest.add(metricName, new Max()); - String nodeResponseName = "node-" + node + ".bytes-received"; + String nodeResponseName = "node-" + connectionId + ".bytes-received"; Sensor nodeResponse = this.metrics.sensor(nodeResponseName); metricName = new MetricName("incoming-byte-rate", metricGrpName, tags); nodeResponse.add(metricName, new Rate()); metricName = new MetricName("response-rate", metricGrpName, "The average number of responses received per second.", tags); nodeResponse.add(metricName, new Rate(new Count())); - String nodeTimeName = "node-" + node + ".latency"; + String nodeTimeName = "node-" + connectionId + ".latency"; Sensor nodeRequestTime = this.metrics.sensor(nodeTimeName); metricName = new MetricName("request-latency-avg", metricGrpName, tags); nodeRequestTime.add(metricName, new Avg()); @@ -626,22 +627,22 @@ public class Selector implements Selectable { } } - public void recordBytesSent(String node, long bytes) { + public void recordBytesSent(String connectionId, long bytes) { long now = time.milliseconds(); this.bytesSent.record(bytes, now); - if (!node.isEmpty()) { - String nodeRequestName = "node-" + node + ".bytes-sent"; + if (!connectionId.isEmpty()) { + String nodeRequestName = "node-" + connectionId + ".bytes-sent"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest != null) nodeRequest.record(bytes, now); } } - public void recordBytesReceived(String node, int bytes) { + public void recordBytesReceived(String connection, int bytes) { long now = time.milliseconds(); this.bytesReceived.record(bytes, now); - if (!node.isEmpty()) { - String nodeRequestName = "node-" + node + ".bytes-received"; + if (!connection.isEmpty()) { + String nodeRequestName = "node-" + connection + ".bytes-received"; Sensor nodeRequest = this.metrics.getSensor(nodeRequestName); if (nodeRequest != null) nodeRequest.record(bytes, now); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 4e2c883..5a49975 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -166,7 +166,7 @@ public class SelectorTest { assertEquals("No disconnects should have occurred.", 0, selector.disconnected().size()); // handle any responses we may have gotten - for (NetworkReceive receive : selector.completedReceives().values()) { + for (NetworkReceive receive : selector.completedReceives()) { String[] pieces = asString(receive).split("-"); assertEquals("Should be in the form 'conn-counter'", 2, pieces.length); assertEquals("Check the source", receive.source(), pieces[0]); @@ -234,14 +234,14 @@ public class SelectorTest { while (selector.completedReceives().isEmpty()) selector.poll(5); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should not be from the muted node", "0", selector.completedReceives().values().iterator().next().source()); + assertEquals("The response should not be from the muted node", "0", selector.completedReceives().get(0).source()); selector.unmute("1"); do { selector.poll(5); } while (selector.completedReceives().isEmpty()); assertEquals("We should have only one response", 1, selector.completedReceives().size()); - assertEquals("The response should be from the previously muted node", "1", selector.completedReceives().values().iterator().next().source()); + assertEquals("The response should be from the previously muted node", "1", selector.completedReceives().get(0).source()); } private String blockingRequest(String node, String s) throws IOException { @@ -249,7 +249,7 @@ public class SelectorTest { selector.poll(1000L); while (true) { selector.poll(1000L); - for (NetworkReceive receive : selector.completedReceives().values()) + for (NetworkReceive receive : selector.completedReceives()) if (receive.source() == node) return asString(receive); } diff --git a/clients/src/test/java/org/apache/kafka/test/MockSelector.java b/clients/src/test/java/org/apache/kafka/test/MockSelector.java index af16129..51eb9d1 100644 --- a/clients/src/test/java/org/apache/kafka/test/MockSelector.java +++ b/clients/src/test/java/org/apache/kafka/test/MockSelector.java @@ -15,9 +15,7 @@ package org.apache.kafka.test; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; @@ -33,7 +31,7 @@ public class MockSelector implements Selectable { private final Time time; private final List initiatedSends = new ArrayList(); private final List completedSends = new ArrayList(); - private final Map completedReceives = new HashMap(); + private final List completedReceives = new ArrayList(); private final List disconnected = new ArrayList(); private final List connected = new ArrayList(); @@ -88,12 +86,12 @@ public class MockSelector implements Selectable { } @Override - public java.util.Map completedReceives() { + public List completedReceives() { return completedReceives; } public void completeReceive(NetworkReceive receive) { - this.completedReceives.put("mock connection id", receive); + this.completedReceives.add(receive); } @Override diff --git a/core/src/main/scala/kafka/api/RequestKeys.scala b/core/src/main/scala/kafka/api/RequestKeys.scala index ef7a86e..155cb65 100644 --- a/core/src/main/scala/kafka/api/RequestKeys.scala +++ b/core/src/main/scala/kafka/api/RequestKeys.scala @@ -20,6 +20,8 @@ package kafka.api import kafka.common.KafkaException import java.nio.ByteBuffer +import kafka.network.InvalidRequestException + object RequestKeys { val ProduceKey: Short = 0 val FetchKey: Short = 1 @@ -59,7 +61,7 @@ object RequestKeys { def deserializerForKey(key: Short): (ByteBuffer) => RequestOrResponse = { keyToNameAndDeserializerMap.get(key) match { case Some(nameAndSerializer) => nameAndSerializer._2 - case None => throw new KafkaException("Wrong request type %d".format(key)) + case None => throw new InvalidRequestException("Wrong request type %d".format(key)) } } } diff --git a/core/src/main/scala/kafka/network/BlockingChannel.scala b/core/src/main/scala/kafka/network/BlockingChannel.scala index 8139d3a..1197259 100644 --- a/core/src/main/scala/kafka/network/BlockingChannel.scala +++ b/core/src/main/scala/kafka/network/BlockingChannel.scala @@ -19,9 +19,10 @@ package kafka.network import java.net.InetSocketAddress import java.nio.channels._ -import kafka.utils.{nonthreadsafe, Logging} + import kafka.api.RequestOrResponse -import org.apache.kafka.common.network.{NetworkSend, NetworkReceive} +import kafka.utils.{Logging, nonthreadsafe} +import org.apache.kafka.common.network.NetworkReceive object BlockingChannel{ diff --git a/core/src/main/scala/kafka/network/Handler.scala b/core/src/main/scala/kafka/network/Handler.scala index 2148e0c..1a7d56e 100644 --- a/core/src/main/scala/kafka/network/Handler.scala +++ b/core/src/main/scala/kafka/network/Handler.scala @@ -17,7 +17,7 @@ package kafka.network -import org.apache.kafka.common.network.{Send, NetworkReceive} +import org.apache.kafka.common.network.{NetworkReceive, Send} private[kafka] object Handler { diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 263e23e..d0fa294 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -17,15 +17,15 @@ package kafka.network +import java.nio.ByteBuffer import java.util.concurrent._ -import kafka.metrics.KafkaMetricsGroup + import com.yammer.metrics.core.Gauge -import java.nio.ByteBuffer import kafka.api._ import kafka.common.TopicAndPartition -import kafka.utils.{Logging, SystemTime} import kafka.message.ByteBufferMessageSet -import java.net._ +import kafka.metrics.KafkaMetricsGroup +import kafka.utils.{Logging, SystemTime} import org.apache.kafka.common.network.Send import org.apache.kafka.common.protocol.{ApiKeys, SecurityProtocol} import org.apache.kafka.common.requests.{AbstractRequest, RequestHeader} diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 25cfcd0..c9aa979 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -17,25 +17,23 @@ package kafka.network +import java.net._ +import java.nio.channels._ import java.util import java.util.concurrent._ import java.util.concurrent.atomic._ -import java.net._ -import java.nio.channels._ -import com.yammer.metrics.core.Meter import kafka.cluster.EndPoint +import kafka.common.KafkaException +import kafka.metrics.KafkaMetricsGroup +import kafka.utils._ import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.common.network.InvalidReceiveException import org.apache.kafka.common.protocol.SecurityProtocol -import org.apache.kafka.common.utils.{Time, SystemTime} +import org.apache.kafka.common.utils.{SystemTime, Time, Utils} import scala.collection._ -import kafka.common.KafkaException -import kafka.metrics.KafkaMetricsGroup -import kafka.utils._ -import org.apache.kafka.common.utils.Utils - /** * An NIO socket server. The threading model is * 1 Acceptor thread that handles new connections @@ -343,22 +341,23 @@ private[kafka] class Processor(val id: Int, shutdownComplete() throw e } + case e: InvalidReceiveException => + // Log warning and continue since Selector already closed the connection + warn("Connection was closed due to invalid receive. Processor will continue handling other connections") } - collection.JavaConversions.mapAsScalaMap(selector.completedReceives).foreach{ case (connectionId, receive) => { + collection.JavaConversions.collectionAsScalaIterable(selector.completedReceives).foreach( receive => { try { val req = RequestChannel.Request(processor = id, connectionId = receive.source, buffer = receive.payload, startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) requestChannel.sendRequest(req) } catch { - case e: InvalidRequestException => { - info("Closing socket connection to %s due to invalid request: %s".format(connectionId, e.getMessage)) - selector.close(connectionId) - } case e: Throwable => { - error("Closing socket for " + connectionId + " because of error", e) - selector.close(connectionId) + case e: Throwable => { + // note that even though we got an exception, we can assume that receive.source is valid. Issues with constructing a valid receive object were handled earlier + error("Closing socket for " + receive.source + " because of error", e) + selector.close(receive.source) } } selector.mute(receive.source) - }} + }) collection.JavaConversions.iterableAsScalaIterable(selector.completedSends()).foreach( send => { val resp = inflightResponses.remove(send.destination()).get @@ -387,7 +386,7 @@ private[kafka] class Processor(val id: Int, selector.unmute(curr.request.connectionId) } case RequestChannel.SendAction => { - trace("Socket server received response to send, registering for write, sending data and registering for read: " + curr) + trace("Socket server received response to send, registering for write and sending data: " + curr) selector.send(curr.responseSend) inflightResponses += (curr.request.connectionId -> curr) } diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 8047f52..e134fab 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -113,13 +113,17 @@ class SocketServerTest extends JUnitSuite { assertEquals(serializedBytes.toSeq, receiveResponse(traceSocket).toSeq) } - @Test(expected = classOf[IOException]) + @Test def tooBigRequestIsRejected() { val tooManyBytes = new Array[Byte](server.maxRequestSize + 1) new Random().nextBytes(tooManyBytes) val socket = connect() sendRequest(socket, 0, tooManyBytes) - receiveResponse(socket) + try { + receiveResponse(socket) + } catch { + case e: IOException => // thats fine + } } @Test -- 2.3.2 (Apple Git-55) From 1ccd3e60c40fe3b213d6121b4b8a704bb425f8ad Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Wed, 20 May 2015 12:53:06 +0300 Subject: [PATCH 20/24] Revert "removed connection-aging for clients" This reverts commit 016669123a370b561b5ac78f8f1cf7bdd958e7d1. --- .../kafka/clients/consumer/ConsumerConfig.java | 11 +++++++- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../kafka/clients/producer/KafkaProducer.java | 2 +- .../kafka/clients/producer/ProducerConfig.java | 7 +++++- .../org/apache/kafka/common/network/Selector.java | 29 +++++++--------------- .../apache/kafka/common/network/SelectorTest.java | 2 +- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index bdff518..55837cc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -156,6 +156,10 @@ public class ConsumerConfig extends AbstractConfig { public static final String VALUE_DESERIALIZER_CLASS_CONFIG = "value.deserializer"; private static final String VALUE_DESERIALIZER_CLASS_DOC = "Deserializer class for value that implements the Deserializer interface."; + /** connections.max.idle.ms */ + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; + public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, @@ -277,7 +281,12 @@ public class ConsumerConfig extends AbstractConfig { .define(VALUE_DESERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, - VALUE_DESERIALIZER_CLASS_DOC); + VALUE_DESERIALIZER_CLASS_DOC) + .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, + Type.LONG, + 10 * 60 * 1000, + Importance.MEDIUM, + CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addDeserializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 79bce3d..d1d1ec1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -473,7 +473,7 @@ public class KafkaConsumer implements Consumer { Map metricsTags = new LinkedHashMap(); metricsTags.put("client-id", clientId); this.client = new NetworkClient( - new Selector(metrics, time, metricGrpPrefix, metricsTags), + new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, metricsTags), this.metadata, clientId, 100, // a fixed large enough value will suffice diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java index e1ee6b2..7891943 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java @@ -229,7 +229,7 @@ public class KafkaProducer implements Producer { this.metadata.update(Cluster.bootstrap(addresses), time.milliseconds()); NetworkClient client = new NetworkClient( - new Selector(this.metrics, time, "producer", metricTags), + new Selector(config.getLong(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), this.metrics, time, "producer", metricTags), this.metadata, clientId, config.getInt(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION), diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index 187d000..7145fb0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -169,6 +169,10 @@ public class ProducerConfig extends AbstractConfig { public static final String VALUE_SERIALIZER_CLASS_CONFIG = "value.serializer"; private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that implements the Serializer interface."; + /** connections.max.idle.ms */ + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; + public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOSTRAP_SERVERS_DOC) .define(BUFFER_MEMORY_CONFIG, Type.LONG, 32 * 1024 * 1024L, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC) @@ -217,7 +221,8 @@ public class ProducerConfig extends AbstractConfig { Importance.LOW, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DOC) .define(KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) - .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC); + .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) + .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, 10 * 60 * 1000, Importance.MEDIUM, CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addSerializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 6962639..579041d 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -67,7 +67,6 @@ import org.slf4j.LoggerFactory; public class Selector implements Selectable { private static final Logger log = LoggerFactory.getLogger(Selector.class); - private static final long UNLIMITED = -1; private final java.nio.channels.Selector nioSelector; private final Map keys; @@ -98,7 +97,7 @@ public class Selector implements Selectable { throw new KafkaException(e); } this.maxReceiveSize = maxReceiveSize; - + this.connectionsMaxIdleNanos = connectionMaxIdleMs * 1000 * 1000; this.time = time; this.metricGrpPrefix = metricGrpPrefix; this.metricTags = metricTags; @@ -109,24 +108,15 @@ public class Selector implements Selectable { this.disconnected = new ArrayList(); this.failedSends = new ArrayList(); this.sensors = new SelectorMetrics(metrics); - - // we only maintain LRU if we plan on aging out idle connections - if (connectionMaxIdleMs != UNLIMITED) { - this.connectionsMaxIdleNanos = connectionMaxIdleMs * 1000 * 1000; - // initial capacity and load factor are default, we set them explicitly because we want to set accessOrder = true - this.lruConnections = new LinkedHashMap(16, .75F, true); - } else { - this.connectionsMaxIdleNanos = connectionMaxIdleMs; - this.lruConnections = null; - } - + // initial capacity and load factor are default, we set them explicitly because we want to set accessOrder = true + this.lruConnections = new LinkedHashMap(16, .75F, true); currentTimeNanos = new SystemTime().nanoseconds(); nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; this.metricsPerConnection = metricsPerConnection; } - public Selector(Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { - this(NetworkReceive.UNLIMITED, UNLIMITED, metrics, time, metricGrpPrefix, metricTags, true); + public Selector(long connectionMaxIdleMS, Metrics metrics, Time time, String metricGrpPrefix, Map metricTags) { + this(NetworkReceive.UNLIMITED, connectionMaxIdleMS, metrics, time, metricGrpPrefix, metricTags, true); } /** @@ -268,8 +258,8 @@ public class Selector implements Selectable { // register all per-connection metrics at once sensors.maybeRegisterConnectionMetrics(transmissions.id); - if (connectionsMaxIdleNanos != UNLIMITED) - lruConnections.put(transmissions.id, currentTimeNanos); + lruConnections.put(transmissions.id, currentTimeNanos); + try { /* complete any connections that have finished their handshake */ if (key.isConnectable()) { @@ -400,7 +390,7 @@ public class Selector implements Selectable { } private void maybeCloseOldestConnection() { - if (connectionsMaxIdleNanos != UNLIMITED && currentTimeNanos > nextIdleCloseCheckTime) { + if (currentTimeNanos > nextIdleCloseCheckTime) { if (lruConnections.isEmpty()) { nextIdleCloseCheckTime = currentTimeNanos + connectionsMaxIdleNanos; } else { @@ -451,8 +441,7 @@ public class Selector implements Selectable { */ public void close(String id) { SelectionKey key = keyForId(id); - if (connectionsMaxIdleNanos != UNLIMITED) - lruConnections.remove(id); + lruConnections.remove(id); SocketChannel channel = channel(key); Transmissions trans = transmissions(key); if (trans != null) { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java index 5a49975..d23b4b6 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SelectorTest.java @@ -46,7 +46,7 @@ public class SelectorTest { public void setup() throws Exception { this.server = new EchoServer(); this.server.start(); - this.selector = new Selector(new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); + this.selector = new Selector(5000, new Metrics(), new MockTime() , "MetricGroup", new LinkedHashMap()); } @After -- 2.3.2 (Apple Git-55) From 4ed0ef99e5c4e14e667529748e72b9206cde524e Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Wed, 20 May 2015 13:33:24 +0300 Subject: [PATCH 21/24] improving exception handling and other minor fixes --- core/src/main/scala/kafka/api/FetchResponse.scala | 4 ++-- core/src/main/scala/kafka/network/SocketServer.scala | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index ca6b571..267d756 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -108,7 +108,7 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti } class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { - override val size = topicData.sizeInBytes + override def size = topicData.sizeInBytes private var sent = 0L @@ -209,7 +209,7 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { - override val size = fetchResponse.sizeInBytes + override def size = fetchResponse.sizeInBytes private var sent = 0L diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index c9aa979..b68af5b 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -17,6 +17,7 @@ package kafka.network +import java.io.IOException import java.net._ import java.nio.channels._ import java.util @@ -30,6 +31,7 @@ import kafka.utils._ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.InvalidReceiveException import org.apache.kafka.common.protocol.SecurityProtocol +import org.apache.kafka.common.protocol.types.SchemaException import org.apache.kafka.common.utils.{SystemTime, Time, Utils} import scala.collection._ @@ -341,6 +343,12 @@ private[kafka] class Processor(val id: Int, shutdownComplete() throw e } + case e: IOException => { + error("Closing processor %s due to IO exception".format(id)) + swallow(closeAll()) + shutdownComplete() + throw e + } case e: InvalidReceiveException => // Log warning and continue since Selector already closed the connection warn("Connection was closed due to invalid receive. Processor will continue handling other connections") @@ -350,7 +358,7 @@ private[kafka] class Processor(val id: Int, val req = RequestChannel.Request(processor = id, connectionId = receive.source, buffer = receive.payload, startTimeMs = time.milliseconds, securityProtocol = SecurityProtocol.PLAINTEXT) requestChannel.sendRequest(req) } catch { - case e: Throwable => { + case e @ (_: InvalidRequestException | _: SchemaException) => { // note that even though we got an exception, we can assume that receive.source is valid. Issues with constructing a valid receive object were handled earlier error("Closing socket for " + receive.source + " because of error", e) selector.close(receive.source) -- 2.3.2 (Apple Git-55) From 9a912be98b1d6cf59a42f743c213f5959329d0b5 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Sun, 24 May 2015 19:49:23 +0300 Subject: [PATCH 22/24] fixes based on Jun and Guozhang comments. Exposed idle metrics as Gauge, changed Send size to long, and fixed an existing issue where Reporters are not actually loaded --- .../apache/kafka/clients/CommonClientConfigs.java | 2 + .../kafka/clients/consumer/ConsumerConfig.java | 8 ++-- .../kafka/clients/producer/ProducerConfig.java | 6 +-- .../kafka/common/network/ByteBufferSend.java | 2 +- .../org/apache/kafka/common/network/MultiSend.java | 4 +- .../java/org/apache/kafka/common/network/Send.java | 2 +- core/src/main/scala/kafka/Kafka.scala | 12 +++--- core/src/main/scala/kafka/api/FetchResponse.scala | 11 ++--- .../main/scala/kafka/network/RequestChannel.scala | 19 ++++---- .../main/scala/kafka/network/SocketServer.scala | 50 ++++++++++++++++------ .../test/scala/unit/kafka/KafkaConfigTest.scala | 17 ++++---- 11 files changed, 81 insertions(+), 52 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java index cf32e4e..2c421f4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java +++ b/clients/src/main/java/org/apache/kafka/clients/CommonClientConfigs.java @@ -55,4 +55,6 @@ public class CommonClientConfigs { public static final String METRIC_REPORTER_CLASSES_CONFIG = "metric.reporters"; public static final String METRIC_REPORTER_CLASSES_DOC = "A list of classes to use as metrics reporters. Implementing the MetricReporter interface allows plugging in classes that will be notified of new metric creation. The JmxReporter is always included to register JMX statistics."; + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; + public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index 55837cc..1e90524 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -157,8 +157,7 @@ public class ConsumerConfig extends AbstractConfig { private static final String VALUE_DESERIALIZER_CLASS_DOC = "Deserializer class for value that implements the Deserializer interface."; /** connections.max.idle.ms */ - public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; - public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG; static { @@ -282,11 +281,12 @@ public class ConsumerConfig extends AbstractConfig { Type.CLASS, Importance.HIGH, VALUE_DESERIALIZER_CLASS_DOC) + /* default is set to be a bit lower than the server default (10 min), to avoid both client and server closing connection at same time */ .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, - 10 * 60 * 1000, + 9 * 60 * 1000, Importance.MEDIUM, - CONNECTIONS_MAX_IDLE_MS_DOC); + CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addDeserializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java index 7145fb0..c7c6250 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java @@ -170,8 +170,7 @@ public class ProducerConfig extends AbstractConfig { private static final String VALUE_SERIALIZER_CLASS_DOC = "Serializer class for value that implements the Serializer interface."; /** connections.max.idle.ms */ - public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = "connections.max.idle.ms"; - public static final String CONNECTIONS_MAX_IDLE_MS_DOC = "Close idle connections after the number of milliseconds specified by this config."; + public static final String CONNECTIONS_MAX_IDLE_MS_CONFIG = CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_CONFIG; static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOSTRAP_SERVERS_DOC) @@ -222,7 +221,8 @@ public class ProducerConfig extends AbstractConfig { MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DOC) .define(KEY_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, KEY_SERIALIZER_CLASS_DOC) .define(VALUE_SERIALIZER_CLASS_CONFIG, Type.CLASS, Importance.HIGH, VALUE_SERIALIZER_CLASS_DOC) - .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, 10 * 60 * 1000, Importance.MEDIUM, CONNECTIONS_MAX_IDLE_MS_DOC); + /* default is set to be a bit lower than the server default (10 min), to avoid both client and server closing connection at same time */ + .define(CONNECTIONS_MAX_IDLE_MS_CONFIG, Type.LONG, 9 * 60 * 1000, Importance.MEDIUM, CommonClientConfigs.CONNECTIONS_MAX_IDLE_MS_DOC); } public static Map addSerializerToConfig(Map configs, diff --git a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java index 3cdc07f..df0e6d5 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ByteBufferSend.java @@ -47,7 +47,7 @@ public class ByteBufferSend implements Send { } @Override - public int size() { + public long size() { return this.size; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java index 88f283f..0e14a39 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java +++ b/clients/src/main/java/org/apache/kafka/common/network/MultiSend.java @@ -38,7 +38,7 @@ public class MultiSend implements Send { private Iterator sendsIterator; private Send current; private boolean doneSends = false; - private int size = 0; + private long size = 0; public MultiSend(String dest, List sends) { this.dest = dest; @@ -50,7 +50,7 @@ public class MultiSend implements Send { } @Override - public int size() { + public long size() { return size; } diff --git a/clients/src/main/java/org/apache/kafka/common/network/Send.java b/clients/src/main/java/org/apache/kafka/common/network/Send.java index a0c07ad..8f6daad 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Send.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Send.java @@ -42,6 +42,6 @@ public interface Send { /** * Size of the send */ - public int size(); + public long size(); } diff --git a/core/src/main/scala/kafka/Kafka.scala b/core/src/main/scala/kafka/Kafka.scala index 9efabaa..6af7b80 100755 --- a/core/src/main/scala/kafka/Kafka.scala +++ b/core/src/main/scala/kafka/Kafka.scala @@ -17,6 +17,8 @@ package kafka +import java.util.Properties + import scala.collection.JavaConversions._ import joptsimple.OptionParser import metrics.KafkaMetricsReporter @@ -26,7 +28,7 @@ import org.apache.kafka.common.utils.Utils object Kafka extends Logging { - def getKafkaConfigFromArgs(args: Array[String]): KafkaConfig = { + def getPropsFromArgs(args: Array[String]): Properties = { val optionParser = new OptionParser val overrideOpt = optionParser.accepts("override", "Optional property that should override values set in server.properties file") .withRequiredArg() @@ -47,14 +49,14 @@ object Kafka extends Logging { props.putAll(CommandLineUtils.parseKeyValueArgs(options.valuesOf(overrideOpt))) } - - KafkaConfig.fromProps(props) + props } def main(args: Array[String]): Unit = { try { - val serverConfig = getKafkaConfigFromArgs(args) - KafkaMetricsReporter.startReporters(new VerifiableProperties(serverConfig.toProps)) + val serverProps = getPropsFromArgs(args) + val serverConfig = KafkaConfig.fromProps(serverProps) + KafkaMetricsReporter.startReporters(new VerifiableProperties(serverProps)) val kafkaServerStartable = new KafkaServerStartable(serverConfig) // attach shutdown handler to catch control-c diff --git a/core/src/main/scala/kafka/api/FetchResponse.scala b/core/src/main/scala/kafka/api/FetchResponse.scala index 267d756..0b6b33a 100644 --- a/core/src/main/scala/kafka/api/FetchResponse.scala +++ b/core/src/main/scala/kafka/api/FetchResponse.scala @@ -108,7 +108,6 @@ case class TopicData(topic: String, partitionData: Map[Int, FetchResponsePartiti } class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { - override def size = topicData.sizeInBytes private var sent = 0L @@ -116,6 +115,8 @@ class TopicDataSend(val dest: String, val topicData: TopicData) extends Send { override def destination: String = dest + override def size = topicData.headerSize + sends.size() + private val buffer = ByteBuffer.allocate(topicData.headerSize) writeShortString(buffer, topicData.topic) buffer.putInt(topicData.partitionData.size) @@ -209,18 +210,18 @@ case class FetchResponse(correlationId: Int, data: Map[TopicAndPartition, FetchR class FetchResponseSend(val dest: String, val fetchResponse: FetchResponse) extends Send { - override def size = fetchResponse.sizeInBytes + private val payloadSize = fetchResponse.sizeInBytes private var sent = 0L - private val sendSize = 4 /* for size */ + size + override def size = 4 /* for size byte */ + payloadSize - override def completed = sent >= sendSize + override def completed = sent >= size override def destination = dest private val buffer = ByteBuffer.allocate(4 /* for size */ + FetchResponse.headerSize) - buffer.putInt(size) + buffer.putInt(payloadSize) buffer.putInt(fetchResponse.correlationId) buffer.putInt(fetchResponse.dataGroupedByTopic.size) // topic count buffer.rewind() diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index d0fa294..db73fb3 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -69,7 +69,14 @@ object RequestChannel extends Logging { buffer = null private val requestLogger = Logger.getLogger("kafka.request.logger") - trace("Processor %d received request : %s".format(processor, requestObj)) + + private val requestDesc = + if (requestObj != null) + requestObj.describe(false) + else + header.toString + " -- " + body.toString + + trace("Processor %d received request : %s".format(processor, requestDesc)) def updateRequestMetrics() { val endTimeMs = SystemTime.milliseconds @@ -101,20 +108,10 @@ object RequestChannel extends Logging { m.totalTimeHist.update(totalTime) } if(requestLogger.isTraceEnabled) { - val requestDesc = - if (requestObj != null) - requestObj.describe(true) - else - header.toString + " -- " + body.toString requestLogger.trace("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" .format(requestDesc, connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) } else if(requestLogger.isDebugEnabled) { - val requestDesc = - if (requestObj != null) - requestObj.describe(false) - else - header.toString + " -- " + body.toString requestLogger.debug("Completed request:%s from connection %s;totalTime:%d,requestQueueTime:%d,localTime:%d,remoteTime:%d,responseQueueTime:%d,sendTime:%d" .format(requestDesc, connectionId, totalTime, requestQueueTime, apiLocalTime, apiRemoteTime, responseQueueTime, responseSendTime)) } diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index b68af5b..788c41e 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -24,10 +24,12 @@ import java.util import java.util.concurrent._ import java.util.concurrent.atomic._ +import com.yammer.metrics.core.Gauge import kafka.cluster.EndPoint import kafka.common.KafkaException import kafka.metrics.KafkaMetricsGroup import kafka.utils._ +import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.InvalidReceiveException import org.apache.kafka.common.protocol.SecurityProtocol @@ -54,7 +56,7 @@ class SocketServer(val brokerId: Int, val maxConnectionsPerIpOverrides: Map[String, Int] ) extends Logging with KafkaMetricsGroup { this.logIdent = "[Socket Server on Broker " + brokerId + "], " private val time = new SystemTime() - private val metrics = new Metrics(time); + private val allMetrics = new Array[Metrics](numProcessorThreads) private val processors = new Array[Processor](numProcessorThreads) private[network] var acceptors = mutable.Map[EndPoint,Acceptor]() val requestChannel = new RequestChannel(numProcessorThreads, maxQueuedRequests) @@ -73,8 +75,27 @@ class SocketServer(val brokerId: Int, def startup() { val quotas = new ConnectionQuotas(maxConnectionsPerIp, maxConnectionsPerIpOverrides) + val aggregateIdleGauge = newGauge("NetworkProcessorAvgIdlePercent", + new Gauge[Double] { + def value = { + var sum: Double = 0 + var count = 0 + for (i <- 0 until numProcessorThreads) { + var metricTags = new util.HashMap[String, String]() + metricTags.put("networkProcessor", i.toString) + + sum += allMetrics(i).metrics().get(new MetricName("io-wait-ratio", "SocketServer-metrics", metricTags)).value() + count += 1 + } + sum / count + } + } + ) + + this.synchronized { for (i <- 0 until numProcessorThreads) { + allMetrics(i) = new Metrics(time); processors(i) = new Processor(i, time, maxRequestSize, @@ -83,7 +104,8 @@ class SocketServer(val brokerId: Int, quotas, connectionsMaxIdleMs, portToProtocol, - metrics) + allMetrics(i) + ) Utils.newThread("kafka-network-thread-%d-%d".format(brokerId, i), processors(i), false).start() } } @@ -310,13 +332,23 @@ private[kafka] class Processor(val id: Int, connectionQuotas: ConnectionQuotas, val connectionsMaxIdleMs: Long, val portToProtocol: ConcurrentHashMap[Int,SecurityProtocol], - val metrics: Metrics) extends AbstractServerThread(connectionQuotas) { + val metrics: Metrics) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { private val newConnections = new ConcurrentLinkedQueue[SocketChannel]() private val inflightResponses = mutable.Map[String, RequestChannel.Response]() private var metricTags = new util.HashMap[String, String]() - metricTags.put("processor-id", id.toString) + metricTags.put("networkProcessor", id.toString) + + newGauge("IdlePercent", + new Gauge[Double] { + def value = { + metrics.metrics().get(new MetricName("io-wait-ratio", "SocketServer-metrics", metricTags)).value() + } + }, + JavaConversions.mapAsScalaMap(metricTags) + ) + private val selector = new org.apache.kafka.common.network.Selector( maxRequestSize, connectionsMaxIdleMs, @@ -337,14 +369,8 @@ private[kafka] class Processor(val id: Int, try { selector.poll(300) } catch { - case e: IllegalStateException => { - error("Closing processor %s due to illegal state".format(id)) - swallow(closeAll()) - shutdownComplete() - throw e - } - case e: IOException => { - error("Closing processor %s due to IO exception".format(id)) + case e @ (_: IllegalStateException | _: IOException) => { + error("Closing processor %s due to illegal state or IO exception".format(id)) swallow(closeAll()) shutdownComplete() throw e diff --git a/core/src/test/scala/unit/kafka/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/KafkaConfigTest.scala index bc4aef3..4cb92de 100644 --- a/core/src/test/scala/unit/kafka/KafkaConfigTest.scala +++ b/core/src/test/scala/unit/kafka/KafkaConfigTest.scala @@ -20,6 +20,7 @@ import java.io.{FileOutputStream, File} import java.security.Permission import kafka.Kafka +import kafka.server.KafkaConfig import org.junit.{After, Before, Test} import junit.framework.Assert._ @@ -57,20 +58,20 @@ class KafkaTest { val propertiesFile = prepareDefaultConfig() // We should load configuration file without any arguments - val config1 = Kafka.getKafkaConfigFromArgs(Array(propertiesFile)) + val config1 = KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile))) assertEquals(1, config1.brokerId) // We should be able to override given property on command line - val config2 = Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "--override", "broker.id=2")) + val config2 = KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "--override", "broker.id=2"))) assertEquals(2, config2.brokerId) // We should be also able to set completely new property - val config3 = Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "--override", "log.cleanup.policy=compact")) + val config3 = KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "--override", "log.cleanup.policy=compact"))) assertEquals(1, config3.brokerId) assertEquals("compact", config3.logCleanupPolicy) // We should be also able to set several properties - val config4 = Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "--override", "log.cleanup.policy=compact", "--override", "broker.id=2")) + val config4 = KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "--override", "log.cleanup.policy=compact", "--override", "broker.id=2"))) assertEquals(2, config4.brokerId) assertEquals("compact", config4.logCleanupPolicy) } @@ -78,25 +79,25 @@ class KafkaTest { @Test(expected = classOf[ExitCalled]) def testGetKafkaConfigFromArgsWrongSetValue(): Unit = { val propertiesFile = prepareDefaultConfig() - Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "--override", "a=b=c")) + KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "--override", "a=b=c"))) } @Test(expected = classOf[ExitCalled]) def testGetKafkaConfigFromArgsNonArgsAtTheEnd(): Unit = { val propertiesFile = prepareDefaultConfig() - Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "--override", "broker.id=1", "broker.id=2")) + KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "--override", "broker.id=1", "broker.id=2"))) } @Test(expected = classOf[ExitCalled]) def testGetKafkaConfigFromArgsNonArgsOnly(): Unit = { val propertiesFile = prepareDefaultConfig() - Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "broker.id=1", "broker.id=2")) + KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "broker.id=1", "broker.id=2"))) } @Test(expected = classOf[ExitCalled]) def testGetKafkaConfigFromArgsNonArgsAtTheBegging(): Unit = { val propertiesFile = prepareDefaultConfig() - Kafka.getKafkaConfigFromArgs(Array(propertiesFile, "broker.id=1", "--override", "broker.id=2")) + KafkaConfig.fromProps(Kafka.getPropsFromArgs(Array(propertiesFile, "broker.id=1", "--override", "broker.id=2"))) } def prepareDefaultConfig(): String = { -- 2.3.2 (Apple Git-55) From 96dc606dd6ec113a72dc012e9ed820c6ae0efd17 Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Sun, 31 May 2015 23:22:59 +0300 Subject: [PATCH 23/24] move to single metrics object --- .../org/apache/kafka/common/network/Selector.java | 23 +++++++---- .../main/scala/kafka/network/RequestChannel.scala | 3 +- .../main/scala/kafka/network/SocketServer.scala | 45 ++++++++++------------ core/src/main/scala/kafka/server/KafkaConfig.scala | 44 +++++++++++++++++++-- core/src/main/scala/kafka/server/KafkaServer.scala | 23 +++++++++-- .../unit/kafka/network/SocketServerTest.scala | 10 ++++- .../kafka/server/KafkaConfigConfigDefTest.scala | 8 ++++ 7 files changed, 114 insertions(+), 42 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index 579041d..b34b5de 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -276,7 +276,7 @@ public class Selector implements Selectable { try { transmissions.receive.readFrom(channel); } catch (InvalidReceiveException e) { - log.error("Illegal state exception when receiving data from " + getConnectionId(key) + " closing connection", e); + log.error("Invalid data received from " + getConnectionId(key) + " closing connection", e); close(transmissions.id); throw e; } @@ -526,20 +526,27 @@ public class Selector implements Selectable { public SelectorMetrics(Metrics metrics) { this.metrics = metrics; String metricGrpName = metricGrpPrefix + "-metrics"; + StringBuilder tagsSuffix = new StringBuilder(); - this.connectionClosed = this.metrics.sensor("connections-closed"); + for (Map.Entry tag: metricTags.entrySet()) { + tagsSuffix.append(tag.getKey()); + tagsSuffix.append("-"); + tagsSuffix.append(tag.getValue()); + } + + this.connectionClosed = this.metrics.sensor("connections-closed:" + tagsSuffix.toString()); MetricName metricName = new MetricName("connection-close-rate", metricGrpName, "Connections closed per second in the window.", metricTags); this.connectionClosed.add(metricName, new Rate()); - this.connectionCreated = this.metrics.sensor("connections-created"); + this.connectionCreated = this.metrics.sensor("connections-created:" + tagsSuffix.toString()); metricName = new MetricName("connection-creation-rate", metricGrpName, "New connections established per second in the window.", metricTags); this.connectionCreated.add(metricName, new Rate()); - this.bytesTransferred = this.metrics.sensor("bytes-sent-received"); + this.bytesTransferred = this.metrics.sensor("bytes-sent-received:" + tagsSuffix.toString()); metricName = new MetricName("network-io-rate", metricGrpName, "The average number of network operations (reads or writes) on all connections per second.", metricTags); bytesTransferred.add(metricName, new Rate(new Count())); - this.bytesSent = this.metrics.sensor("bytes-sent", bytesTransferred); + this.bytesSent = this.metrics.sensor("bytes-sent:" + tagsSuffix.toString(), bytesTransferred); metricName = new MetricName("outgoing-byte-rate", metricGrpName, "The average number of outgoing bytes sent per second to all servers.", metricTags); this.bytesSent.add(metricName, new Rate()); metricName = new MetricName("request-rate", metricGrpName, "The average number of requests sent per second.", metricTags); @@ -549,13 +556,13 @@ public class Selector implements Selectable { metricName = new MetricName("request-size-max", metricGrpName, "The maximum size of any request sent in the window.", metricTags); this.bytesSent.add(metricName, new Max()); - this.bytesReceived = this.metrics.sensor("bytes-received", bytesTransferred); + this.bytesReceived = this.metrics.sensor("bytes-received:" + tagsSuffix.toString(), bytesTransferred); metricName = new MetricName("incoming-byte-rate", metricGrpName, "Bytes/second read off all sockets", metricTags); this.bytesReceived.add(metricName, new Rate()); metricName = new MetricName("response-rate", metricGrpName, "Responses received sent per second.", metricTags); this.bytesReceived.add(metricName, new Rate(new Count())); - this.selectTime = this.metrics.sensor("select-time"); + this.selectTime = this.metrics.sensor("select-time:" + tagsSuffix.toString()); metricName = new MetricName("select-rate", metricGrpName, "Number of times the I/O layer checked for new I/O to perform per second", metricTags); this.selectTime.add(metricName, new Rate(new Count())); metricName = new MetricName("io-wait-time-ns-avg", metricGrpName, "The average length of time the I/O thread spent waiting for a socket ready for reads or writes in nanoseconds.", metricTags); @@ -563,7 +570,7 @@ public class Selector implements Selectable { metricName = new MetricName("io-wait-ratio", metricGrpName, "The fraction of time the I/O thread spent waiting.", metricTags); this.selectTime.add(metricName, new Rate(TimeUnit.NANOSECONDS)); - this.ioTime = this.metrics.sensor("io-time"); + this.ioTime = this.metrics.sensor("io-time:" + tagsSuffix.toString()); metricName = new MetricName("io-time-ns-avg", metricGrpName, "The average length of time for I/O per select call in nanoseconds.", metricTags); this.ioTime.add(metricName, new Avg()); metricName = new MetricName("io-ratio", metricGrpName, "The fraction of time the I/O thread spent doing I/O", metricTags); diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index db73fb3..ca37ee8 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -70,11 +70,12 @@ object RequestChannel extends Logging { buffer = null private val requestLogger = Logger.getLogger("kafka.request.logger") - private val requestDesc = + private def requestDesc { if (requestObj != null) requestObj.describe(false) else header.toString + " -- " + body.toString + } trace("Processor %d received request : %s".format(processor, requestDesc)) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 788c41e..4814695 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -53,14 +53,21 @@ class SocketServer(val brokerId: Int, val maxRequestSize: Int = Int.MaxValue, val maxConnectionsPerIp: Int = Int.MaxValue, val connectionsMaxIdleMs: Long, - val maxConnectionsPerIpOverrides: Map[String, Int] ) extends Logging with KafkaMetricsGroup { + val maxConnectionsPerIpOverrides: Map[String, Int], + val time: Time, + val metrics: Metrics) extends Logging with KafkaMetricsGroup { this.logIdent = "[Socket Server on Broker " + brokerId + "], " - private val time = new SystemTime() - private val allMetrics = new Array[Metrics](numProcessorThreads) + private val processors = new Array[Processor](numProcessorThreads) private[network] var acceptors = mutable.Map[EndPoint,Acceptor]() val requestChannel = new RequestChannel(numProcessorThreads, maxQueuedRequests) + private val allMetricNames = (0 until numProcessorThreads).map { i => + val tags = new util.HashMap[String, String]() + tags.put("networkProcessor", i.toString) + new MetricName("io-wait-ratio", "socket-server-metrics", tags) + } + /* I'm pushing the mapping of port-to-protocol to the processor level, so the processor can put the correct protocol in the request channel. we'll probably have a more elegant way of doing this once we patch the request channel @@ -75,27 +82,16 @@ class SocketServer(val brokerId: Int, def startup() { val quotas = new ConnectionQuotas(maxConnectionsPerIp, maxConnectionsPerIpOverrides) - val aggregateIdleGauge = newGauge("NetworkProcessorAvgIdlePercent", + newGauge("NetworkProcessorAvgIdlePercent", new Gauge[Double] { - def value = { - var sum: Double = 0 - var count = 0 - for (i <- 0 until numProcessorThreads) { - var metricTags = new util.HashMap[String, String]() - metricTags.put("networkProcessor", i.toString) - - sum += allMetrics(i).metrics().get(new MetricName("io-wait-ratio", "SocketServer-metrics", metricTags)).value() - count += 1 - } - sum / count - } + def value = allMetricNames.map( metricName => + metrics.metrics().get(metricName).value()).sum / numProcessorThreads } ) this.synchronized { for (i <- 0 until numProcessorThreads) { - allMetrics(i) = new Metrics(time); processors(i) = new Processor(i, time, maxRequestSize, @@ -104,7 +100,7 @@ class SocketServer(val brokerId: Int, quotas, connectionsMaxIdleMs, portToProtocol, - allMetrics(i) + metrics ) Utils.newThread("kafka-network-thread-%d-%d".format(brokerId, i), processors(i), false).start() } @@ -245,8 +241,9 @@ private[kafka] class Acceptor(val host: String, try { key = iter.next iter.remove() - if(key.isAcceptable) - accept(key, processors(currentProcessor)) + if(key.isAcceptable) { + accept(key, processors(currentProcessor)) + } else throw new IllegalStateException("Unrecognized key state for acceptor thread.") @@ -278,7 +275,7 @@ private[kafka] class Acceptor(val host: String, serverChannel.socket().setReceiveBufferSize(recvBufferSize) try { serverChannel.socket.bind(socketAddress) - info("Awaiting socket connections on %s:%d.".format(socketAddress.getHostName, port)) + info("Awaiting socket connections on %s:%d.".format(socketAddress.getHostName, serverChannel.socket.getLocalPort)) } catch { case e: SocketException => throw new KafkaException("Socket server failed to bind to %s:%d: %s.".format(socketAddress.getHostName, port, e.getMessage), e) @@ -337,13 +334,13 @@ private[kafka] class Processor(val id: Int, private val newConnections = new ConcurrentLinkedQueue[SocketChannel]() private val inflightResponses = mutable.Map[String, RequestChannel.Response]() - private var metricTags = new util.HashMap[String, String]() + private val metricTags = new util.HashMap[String, String]() metricTags.put("networkProcessor", id.toString) newGauge("IdlePercent", new Gauge[Double] { def value = { - metrics.metrics().get(new MetricName("io-wait-ratio", "SocketServer-metrics", metricTags)).value() + metrics.metrics().get(new MetricName("io-wait-ratio", "socket-server-metrics", metricTags)).value() } }, JavaConversions.mapAsScalaMap(metricTags) @@ -354,7 +351,7 @@ private[kafka] class Processor(val id: Int, connectionsMaxIdleMs, metrics, time, - "SocketServer", + "socket-server", metricTags, false) diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 9efa15c..2afe48f 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -24,9 +24,11 @@ import kafka.cluster.EndPoint import kafka.consumer.ConsumerConfig import kafka.message.{BrokerCompressionCodec, CompressionCodec, Message, MessageSet} import kafka.utils.CoreUtils +import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.metrics.MetricsReporter import org.apache.kafka.common.protocol.SecurityProtocol -import scala.collection.{immutable, JavaConversions, Map} +import scala.collection.{mutable, immutable, JavaConversions, Map} object Defaults { /** ********* Zookeeper Configuration ***********/ @@ -126,6 +128,10 @@ object Defaults { val DeleteTopicEnable = false val CompressionType = "producer" + + val MetricNumSamples = 2 + val MetricSampleWindowMs = 1000 + val MetricReporterClasses = "" } object KafkaConfig { @@ -233,6 +239,10 @@ object KafkaConfig { val DeleteTopicEnableProp = "delete.topic.enable" val CompressionTypeProp = "compression.type" + val MetricSampleWindowMsProp = CommonClientConfigs.METRICS_SAMPLE_WINDOW_MS_CONFIG + val MetricNumSamplesProp: String = CommonClientConfigs.METRICS_NUM_SAMPLES_CONFIG + val MetricReporterClassesProp: String = CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG + /* Documentation */ /** ********* Zookeeper Configuration ***********/ @@ -364,6 +374,10 @@ object KafkaConfig { "('gzip', 'snappy', lz4). It additionally accepts 'uncompressed' which is equivalent to no compression; and " + "'producer' which means retain the original compression codec set by the producer." + val MetricSampleWindowMsDoc = CommonClientConfigs.METRICS_SAMPLE_WINDOW_MS_DOC + val MetricNumSamplesDoc = CommonClientConfigs.METRICS_NUM_SAMPLES_DOC + val MetricReporterClassesDoc = CommonClientConfigs.METRIC_REPORTER_CLASSES_DOC + private val configDef = { import ConfigDef.Range._ @@ -479,6 +493,9 @@ object KafkaConfig { .define(OffsetCommitRequiredAcksProp, SHORT, Defaults.OffsetCommitRequiredAcks, HIGH, OffsetCommitRequiredAcksDoc) .define(DeleteTopicEnableProp, BOOLEAN, Defaults.DeleteTopicEnable, HIGH, DeleteTopicEnableDoc) .define(CompressionTypeProp, STRING, Defaults.CompressionType, HIGH, CompressionTypeDoc) + .define(MetricNumSamplesProp, INT, Defaults.MetricNumSamples, atLeast(1), LOW, MetricNumSamplesDoc) + .define(MetricSampleWindowMsProp, LONG, Defaults.MetricSampleWindowMs, atLeast(1), LOW, MetricSampleWindowMsDoc) + .define(MetricReporterClassesProp, STRING, Defaults.MetricReporterClasses, LOW, MetricReporterClassesDoc) } def configNames() = { @@ -598,7 +615,10 @@ object KafkaConfig { offsetCommitTimeoutMs = parsed.get(OffsetCommitTimeoutMsProp).asInstanceOf[Int], offsetCommitRequiredAcks = parsed.get(OffsetCommitRequiredAcksProp).asInstanceOf[Short], deleteTopicEnable = parsed.get(DeleteTopicEnableProp).asInstanceOf[Boolean], - compressionType = parsed.get(CompressionTypeProp).asInstanceOf[String] + compressionType = parsed.get(CompressionTypeProp).asInstanceOf[String], + metricNumSamples = parsed.get(MetricNumSamplesProp).asInstanceOf[Int], + metricSampleWindowMs = parsed.get(MetricSampleWindowMsProp).asInstanceOf[Long], + _metricReporterClasses = parsed.get(MetricReporterClassesProp).asInstanceOf[String] ) } @@ -633,7 +653,7 @@ object KafkaConfig { } } -class KafkaConfig(/** ********* Zookeeper Configuration ***********/ +class KafkaConfig (/** ********* Zookeeper Configuration ***********/ val zkConnect: String, val zkSessionTimeoutMs: Int = Defaults.ZkSessionTimeoutMs, private val _zkConnectionTimeoutMs: Option[Int] = None, @@ -742,7 +762,11 @@ class KafkaConfig(/** ********* Zookeeper Configuration ***********/ val offsetCommitRequiredAcks: Short = Defaults.OffsetCommitRequiredAcks, val deleteTopicEnable: Boolean = Defaults.DeleteTopicEnable, - val compressionType: String = Defaults.CompressionType + val compressionType: String = Defaults.CompressionType, + + val metricSampleWindowMs: Long = Defaults.MetricSampleWindowMs, + val metricNumSamples: Int = Defaults.MetricNumSamples, + private val _metricReporterClasses: String = Defaults.MetricReporterClasses ) { val zkConnectionTimeoutMs: Int = _zkConnectionTimeoutMs.getOrElse(zkSessionTimeoutMs) @@ -762,6 +786,8 @@ class KafkaConfig(/** ********* Zookeeper Configuration ***********/ val maxConnectionsPerIpOverrides: Map[String, Int] = getMap(KafkaConfig.MaxConnectionsPerIpOverridesProp, _maxConnectionsPerIpOverrides).map { case (k, v) => (k, v.toInt)} + val metricReporterClasses: Seq[MetricsReporter] = getMetricClasses(_metricReporterClasses) + private def getLogRetentionTimeMillis: Long = { val millisInMinute = 60L * 1000L val millisInHour = 60L * millisInMinute @@ -826,6 +852,13 @@ class KafkaConfig(/** ********* Zookeeper Configuration ***********/ } } + private def getMetricClasses(metricClasses: String): Seq[MetricsReporter] = { + val metricReporterNameList = CoreUtils.parseCsvList(metricClasses) + metricReporterNameList.map(reporterName => { + CoreUtils.createObject[MetricsReporter](reporterName) + }) + } + validateValues() @@ -964,6 +997,9 @@ class KafkaConfig(/** ********* Zookeeper Configuration ***********/ props.put(OffsetCommitRequiredAcksProp, offsetCommitRequiredAcks.toString) props.put(DeleteTopicEnableProp, deleteTopicEnable.toString) props.put(CompressionTypeProp, compressionType.toString) + props.put(MetricNumSamplesProp, metricNumSamples.toString) + props.put(MetricSampleWindowMsProp, metricSampleWindowMs.toString) + props.put(MetricReporterClassesProp, _metricReporterClasses) props } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index c812ac8..6aa6a6f 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -21,14 +21,15 @@ import kafka.admin._ import kafka.log.LogConfig import kafka.log.CleanerConfig import kafka.log.LogManager -import kafka.utils._ import java.util.concurrent._ import atomic.{AtomicInteger, AtomicBoolean} import java.io.File +import kafka.utils._ +import org.apache.kafka.common.metrics._ import org.apache.kafka.common.network.NetworkReceive -import collection.mutable +import scala.collection.{JavaConversions, mutable} import org.I0Itec.zkclient.ZkClient import kafka.controller.{ControllerStats, KafkaController} import kafka.cluster.{EndPoint, Broker} @@ -50,6 +51,17 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg private var shutdownLatch = new CountDownLatch(1) + private val metricConfig: MetricConfig = new MetricConfig() + .samples(config.metricNumSamples) + .timeWindow(config.metricSampleWindowMs, TimeUnit.MILLISECONDS) + private val jmxPrefix: String = "kafka.producer" + private val reporters: java.util.List[MetricsReporter] = JavaConversions.seqAsJavaList(new JmxReporter(jmxPrefix) +: config.metricReporterClasses) + + + // This exists so SocketServer (which uses Client libraries) can use the client Time objects without having to convert all of Kafka to use them + // Once we get rid of kafka.utils.time, we can get rid of this too + private val socketServerTime: org.apache.kafka.common.utils.Time = new org.apache.kafka.common.utils.SystemTime() + val brokerState: BrokerState = new BrokerState var apis: KafkaApis = null @@ -119,6 +131,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg config.brokerId = getBrokerId this.logIdent = "[Kafka Server " + config.brokerId + "], " + val metrics = new Metrics(metricConfig, reporters, socketServerTime) + + socketServer = new SocketServer(config.brokerId, config.listeners, config.numNetworkThreads, @@ -128,7 +143,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg config.socketRequestMaxBytes, config.maxConnectionsPerIp, config.connectionsMaxIdleMs, - config.maxConnectionsPerIpOverrides) + config.maxConnectionsPerIpOverrides, + socketServerTime, + metrics) socketServer.startup() /* start replica manager */ diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index e134fab..7dc2fad 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -20,8 +20,10 @@ package kafka.network; import java.net._ import java.io._ import kafka.cluster.EndPoint +import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.NetworkSend import org.apache.kafka.common.protocol.SecurityProtocol +import org.apache.kafka.common.utils.SystemTime import org.junit._ import org.scalatest.junit.JUnitSuite import java.util.Random @@ -47,7 +49,9 @@ class SocketServerTest extends JUnitSuite { maxRequestSize = 50, maxConnectionsPerIp = 5, connectionsMaxIdleMs = 60*1000, - maxConnectionsPerIpOverrides = Map.empty[String,Int]) + maxConnectionsPerIpOverrides = Map.empty[String,Int], + new SystemTime(), + new Metrics()) server.startup() def sendRequest(socket: Socket, id: Short, request: Array[Byte]) { @@ -183,7 +187,9 @@ class SocketServerTest extends JUnitSuite { maxRequestSize = 50, maxConnectionsPerIp = 5, connectionsMaxIdleMs = 60*1000, - maxConnectionsPerIpOverrides = overrides) + maxConnectionsPerIpOverrides = overrides, + new SystemTime(), + new Metrics()) overrideServer.startup() // make the maximum allowable number of connections and then leak them val conns = ((0 until overrideNum).map(i => connect(overrideServer))) diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigConfigDefTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigConfigDefTest.scala index 8014a5a..7d4e68f 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaConfigConfigDefTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaConfigConfigDefTest.scala @@ -242,6 +242,10 @@ class KafkaConfigConfigDefTest extends JUnit3Suite { //BrokerCompressionCodec.isValid(compressionType) case KafkaConfig.CompressionTypeProp => expected.setProperty(name, randFrom(BrokerCompressionCodec.brokerCompressionOptions)) + case KafkaConfig.MetricNumSamplesProp => expected.setProperty(name, "2") + case KafkaConfig.MetricSampleWindowMsProp => expected.setProperty(name, "1000") + case KafkaConfig.MetricReporterClassesProp => expected.setProperty(name, "") + case nonNegativeIntProperty => expected.setProperty(name, nextInt(Int.MaxValue).toString) } }) @@ -343,6 +347,10 @@ class KafkaConfigConfigDefTest extends JUnit3Suite { case KafkaConfig.DeleteTopicEnableProp => assertPropertyInvalid(getBaseProperties(), name, "not_a_boolean", "0") + case KafkaConfig.MetricNumSamplesProp => assertPropertyInvalid(getBaseProperties, name, "not_a_number", "-1", "0") + case KafkaConfig.MetricSampleWindowMsProp => assertPropertyInvalid(getBaseProperties, name, "not_a_number", "-1", "0") + case KafkaConfig.MetricReporterClassesProp => // ignore string + case nonNegativeIntProperty => assertPropertyInvalid(getBaseProperties(), name, "not_a_number", "-1") } }) -- 2.3.2 (Apple Git-55) From bdc080c8490f62b6342a6d133ad6d7614d24bf7d Mon Sep 17 00:00:00 2001 From: Gwen Shapira Date: Wed, 3 Jun 2015 15:40:34 -0700 Subject: [PATCH 24/24] improvements on new configs --- .../org/apache/kafka/common/network/Selector.java | 11 +------- .../main/scala/kafka/network/RequestChannel.scala | 2 +- .../main/scala/kafka/network/SocketServer.scala | 3 +- core/src/main/scala/kafka/server/KafkaApis.scala | 2 +- core/src/main/scala/kafka/server/KafkaConfig.scala | 32 +++++++++++++++------- core/src/main/scala/kafka/server/KafkaServer.scala | 6 ++-- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/network/Selector.java b/clients/src/main/java/org/apache/kafka/common/network/Selector.java index b34b5de..effb1e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/Selector.java +++ b/clients/src/main/java/org/apache/kafka/common/network/Selector.java @@ -276,7 +276,7 @@ public class Selector implements Selectable { try { transmissions.receive.readFrom(channel); } catch (InvalidReceiveException e) { - log.error("Invalid data received from " + getConnectionId(key) + " closing connection", e); + log.error("Invalid data received from " + transmissions.id + " closing connection", e); close(transmissions.id); throw e; } @@ -320,15 +320,6 @@ public class Selector implements Selectable { maybeCloseOldestConnection(); } - private String getConnectionId(SelectionKey key) { - SocketChannel channel = (SocketChannel) key.channel(); - String localHost = channel.socket().getLocalAddress().getHostAddress(); - int localPort = channel.socket().getLocalPort(); - String remoteHost = channel.socket().getInetAddress().getHostAddress(); - int remotePort = channel.socket().getPort(); - return localHost + ":" + localPort + "-" + remoteHost + ":" + remotePort; - } - private String socketDescription(SocketChannel channel) { Socket socket = channel.socket(); if (socket == null) diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index ca37ee8..1f67a98 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -70,7 +70,7 @@ object RequestChannel extends Logging { buffer = null private val requestLogger = Logger.getLogger("kafka.request.logger") - private def requestDesc { + private def requestDesc: String = { if (requestObj != null) requestObj.describe(false) else diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 4814695..91319fa 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -241,9 +241,8 @@ private[kafka] class Acceptor(val host: String, try { key = iter.next iter.remove() - if(key.isAcceptable) { + if(key.isAcceptable) accept(key, processors(currentProcessor)) - } else throw new IllegalStateException("Unrecognized key state for acceptor thread.") diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index ebca1e3..d63bc18 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -552,7 +552,7 @@ class KafkaApis(val requestChannel: RequestChannel, // the callback for sending a join-group response def sendResponseCallback(partitions: Set[TopicAndPartition], consumerId: String, generationId: Int, errorCode: Short) { val partitionList = partitions.map(tp => new TopicPartition(tp.topic, tp.partition)).toBuffer - val responseBody = new JoinGroupResponse(errorCode, generationId, joinGroupRequest.consumerId, partitionList) + val responseBody = new JoinGroupResponse(errorCode, generationId, consumerId, partitionList) trace("Sending join group response %s for correlation id %d to client %s." .format(responseBody, request.header.correlationId, request.header.clientId)) requestChannel.sendResponse(new RequestChannel.Response(request, new ResponseSend(request.connectionId, respHeader, responseBody))) diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 2afe48f..534ecec 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -17,6 +17,7 @@ package kafka.server +import java.util import java.util.Properties import kafka.api.ApiVersion @@ -495,7 +496,7 @@ object KafkaConfig { .define(CompressionTypeProp, STRING, Defaults.CompressionType, HIGH, CompressionTypeDoc) .define(MetricNumSamplesProp, INT, Defaults.MetricNumSamples, atLeast(1), LOW, MetricNumSamplesDoc) .define(MetricSampleWindowMsProp, LONG, Defaults.MetricSampleWindowMs, atLeast(1), LOW, MetricSampleWindowMsDoc) - .define(MetricReporterClassesProp, STRING, Defaults.MetricReporterClasses, LOW, MetricReporterClassesDoc) + .define(MetricReporterClassesProp, LIST, Defaults.MetricReporterClasses, LOW, MetricReporterClassesDoc) } def configNames() = { @@ -618,7 +619,7 @@ object KafkaConfig { compressionType = parsed.get(CompressionTypeProp).asInstanceOf[String], metricNumSamples = parsed.get(MetricNumSamplesProp).asInstanceOf[Int], metricSampleWindowMs = parsed.get(MetricSampleWindowMsProp).asInstanceOf[Long], - _metricReporterClasses = parsed.get(MetricReporterClassesProp).asInstanceOf[String] + _metricReporterClasses = parsed.get(MetricReporterClassesProp).asInstanceOf[java.util.List[String]] ) } @@ -766,7 +767,7 @@ class KafkaConfig (/** ********* Zookeeper Configuration ***********/ val metricSampleWindowMs: Long = Defaults.MetricSampleWindowMs, val metricNumSamples: Int = Defaults.MetricNumSamples, - private val _metricReporterClasses: String = Defaults.MetricReporterClasses + private val _metricReporterClasses: java.util.List[String] = util.Arrays.asList(Defaults.MetricReporterClasses) ) { val zkConnectionTimeoutMs: Int = _zkConnectionTimeoutMs.getOrElse(zkSessionTimeoutMs) @@ -786,7 +787,7 @@ class KafkaConfig (/** ********* Zookeeper Configuration ***********/ val maxConnectionsPerIpOverrides: Map[String, Int] = getMap(KafkaConfig.MaxConnectionsPerIpOverridesProp, _maxConnectionsPerIpOverrides).map { case (k, v) => (k, v.toInt)} - val metricReporterClasses: Seq[MetricsReporter] = getMetricClasses(_metricReporterClasses) + val metricReporterClasses: java.util.List[MetricsReporter] = getMetricClasses(_metricReporterClasses) private def getLogRetentionTimeMillis: Long = { val millisInMinute = 60L * 1000L @@ -852,11 +853,22 @@ class KafkaConfig (/** ********* Zookeeper Configuration ***********/ } } - private def getMetricClasses(metricClasses: String): Seq[MetricsReporter] = { - val metricReporterNameList = CoreUtils.parseCsvList(metricClasses) - metricReporterNameList.map(reporterName => { - CoreUtils.createObject[MetricsReporter](reporterName) - }) + private def getMetricClasses(metricClasses: java.util.List[String]): java.util.List[MetricsReporter] = { + + val reporterList = new util.ArrayList[MetricsReporter](); + val iterator = metricClasses.iterator() + + while (iterator.hasNext) { + val reporterName = iterator.next() + if (!reporterName.isEmpty) { + val reporter: MetricsReporter = CoreUtils.createObject[MetricsReporter](reporterName) + reporter.configure(toProps.asInstanceOf[java.util.Map[String, _]]) + reporterList.add(reporter) + } + } + + reporterList + } @@ -999,7 +1011,7 @@ class KafkaConfig (/** ********* Zookeeper Configuration ***********/ props.put(CompressionTypeProp, compressionType.toString) props.put(MetricNumSamplesProp, metricNumSamples.toString) props.put(MetricSampleWindowMsProp, metricSampleWindowMs.toString) - props.put(MetricReporterClassesProp, _metricReporterClasses) + props.put(MetricReporterClassesProp, JavaConversions.collectionAsScalaIterable(_metricReporterClasses).mkString(",")) props } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 44fe8b3..b320ce9 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -54,8 +54,10 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg private val metricConfig: MetricConfig = new MetricConfig() .samples(config.metricNumSamples) .timeWindow(config.metricSampleWindowMs, TimeUnit.MILLISECONDS) - private val jmxPrefix: String = "kafka.producer" - private val reporters: java.util.List[MetricsReporter] = JavaConversions.seqAsJavaList(new JmxReporter(jmxPrefix) +: config.metricReporterClasses) + private val jmxPrefix: String = "kafka.server" + private val reporters: java.util.List[MetricsReporter] = config.metricReporterClasses + reporters.add(new JmxReporter(jmxPrefix)) + // This exists so SocketServer (which uses Client libraries) can use the client Time objects without having to convert all of Kafka to use them -- 2.3.2 (Apple Git-55)