diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java index 1c05a77..1c18832 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLContext.java @@ -16,27 +16,25 @@ */ package kafka.etl; + import java.io.IOException; import java.net.URI; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.zip.CRC32; import kafka.api.FetchRequest; -import kafka.javaapi.MultiFetchResponse; import kafka.api.OffsetRequest; import kafka.common.ErrorMapping; +import kafka.javaapi.MultiFetchResponse; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.Message; import kafka.message.MessageAndOffset; -import kafka.message.MessageSet; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.lib.MultipleOutputs; -import java.nio.ByteBuffer; @SuppressWarnings({ "deprecation"}) public class KafkaETLContext { @@ -139,7 +137,7 @@ public class KafkaETLContext { while ( !gotNext && _respIterator.hasNext()) { ByteBufferMessageSet msgSet = _respIterator.next(); if ( hasError(msgSet)) return false; - _messageIt = (Iterator) msgSet.iterator(); + _messageIt = msgSet.iterator(); gotNext = get(key, value); } } @@ -190,17 +188,17 @@ public class KafkaETLContext { protected boolean get(KafkaETLKey key, BytesWritable value) throws IOException { if (_messageIt != null && _messageIt.hasNext()) { - MessageAndOffset msgAndOffset = _messageIt.next(); + MessageAndOffset messageAndOffset = _messageIt.next(); - ByteBuffer buf = msgAndOffset.message().payload(); + ByteBuffer buf = messageAndOffset.message().payload(); int origSize = buf.remaining(); byte[] bytes = new byte[origSize]; - buf.get(bytes, buf.position(), origSize); + buf.get(bytes, buf.position(), origSize); value.set(bytes, 0, origSize); - key.set(_index, _offset, msgAndOffset.message().checksum()); + key.set(_index, _offset, messageAndOffset.message().checksum()); - _offset = msgAndOffset.offset(); //increase offset + _offset = messageAndOffset.offset(); //increase offset _count ++; //increase count return true; diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLInputFormat.java contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLInputFormat.java index 69aafed..ddd6b72 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLInputFormat.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLInputFormat.java @@ -16,6 +16,7 @@ */ package kafka.etl; + import java.io.IOException; import java.net.URI; import java.util.Map; @@ -23,13 +24,13 @@ import kafka.consumer.SimpleConsumer; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.mapred.lib.MultipleOutputs; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.SequenceFileInputFormat; +import org.apache.hadoop.mapred.lib.MultipleOutputs; @SuppressWarnings("deprecation") diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLJob.java contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLJob.java index 5b8e77d..1a4bcba 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLJob.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLJob.java @@ -16,13 +16,13 @@ */ package kafka.etl; + import java.net.URI; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.mapred.lib.MultipleOutputs; diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLKey.java contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLKey.java index e448366..aafecea 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLKey.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLKey.java @@ -16,11 +16,11 @@ */ package kafka.etl; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.io.WritableComparable; -import kafka.etl.KafkaETLKey; public class KafkaETLKey implements WritableComparable{ diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLUtils.java contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLUtils.java index 9691b94..02d79a1 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLUtils.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/KafkaETLUtils.java @@ -17,6 +17,7 @@ package kafka.etl; + import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -33,7 +34,6 @@ import java.util.Arrays; import java.util.Enumeration; import java.util.List; import java.util.Properties; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; diff --git contrib/hadoop-consumer/src/main/java/kafka/etl/impl/DataGenerator.java contrib/hadoop-consumer/src/main/java/kafka/etl/impl/DataGenerator.java index ba9646b..5166358 100644 --- contrib/hadoop-consumer/src/main/java/kafka/etl/impl/DataGenerator.java +++ contrib/hadoop-consumer/src/main/java/kafka/etl/impl/DataGenerator.java @@ -17,32 +17,24 @@ package kafka.etl.impl; -import java.io.IOException; + import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; import java.util.List; -import java.util.Random; -import java.util.Map.Entry; import java.util.Properties; - -import kafka.message.NoCompressionCodec; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.mapred.JobConf; - +import java.util.Random; import kafka.etl.KafkaETLKey; import kafka.etl.KafkaETLRequest; -import kafka.etl.KafkaETLUtils; import kafka.etl.Props; import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.Message; import kafka.javaapi.producer.SyncProducer; +import kafka.message.Message; import kafka.producer.SyncProducerConfig; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.BytesWritable; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.mapred.JobConf; /** * Use this class to produce test events to Kafka server. Each event contains a diff --git contrib/hadoop-producer/src/main/java/kafka/bridge/examples/TextPublisher.java contrib/hadoop-producer/src/main/java/kafka/bridge/examples/TextPublisher.java index 815c13f..5acbcee 100644 --- contrib/hadoop-producer/src/main/java/kafka/bridge/examples/TextPublisher.java +++ contrib/hadoop-producer/src/main/java/kafka/bridge/examples/TextPublisher.java @@ -16,8 +16,9 @@ */ package kafka.bridge.examples; -import kafka.bridge.hadoop.KafkaOutputFormat; +import java.io.IOException; +import kafka.bridge.hadoop.KafkaOutputFormat; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; @@ -27,8 +28,6 @@ import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; -import java.io.IOException; - public class TextPublisher { public static void main(String[] args) throws Exception diff --git contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaOutputFormat.java contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaOutputFormat.java index 5733267..4b9343f 100644 --- contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaOutputFormat.java +++ contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaOutputFormat.java @@ -16,24 +16,26 @@ */ package kafka.bridge.hadoop; -import java.util.Properties; +import java.io.IOException; +import java.net.URI; +import java.util.Properties; import kafka.javaapi.producer.Producer; import kafka.message.Message; import kafka.producer.ProducerConfig; - -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.log4j.Logger; -import java.io.IOException; -import java.net.URI; - public class KafkaOutputFormat extends OutputFormat { private Logger log = Logger.getLogger(KafkaOutputFormat.class); diff --git contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaRecordWriter.java contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaRecordWriter.java index efb2b5e..af9c650 100644 --- contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaRecordWriter.java +++ contrib/hadoop-producer/src/main/java/kafka/bridge/hadoop/KafkaRecordWriter.java @@ -16,19 +16,18 @@ */ package kafka.bridge.hadoop; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import kafka.javaapi.producer.Producer; import kafka.javaapi.producer.ProducerData; import kafka.message.Message; - import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; - public class KafkaRecordWriter extends RecordWriter { protected Producer producer; diff --git contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java index cac634b..faa1950 100644 --- contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java +++ contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java @@ -16,9 +16,12 @@ */ package kafka.bridge.pig; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import kafka.bridge.hadoop.KafkaOutputFormat; import kafka.bridge.hadoop.KafkaRecordWriter; - import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.Encoder; import org.apache.hadoop.fs.Path; @@ -33,10 +36,6 @@ import org.apache.pig.data.Tuple; import org.apache.pig.piggybank.storage.avro.PigAvroDatumWriter; import org.apache.pig.piggybank.storage.avro.PigSchema2Avro; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; - public class AvroKafkaStorage extends StoreFunc { protected KafkaRecordWriter writer; diff --git core/src/main/scala/kafka/Kafka.scala core/src/main/scala/kafka/Kafka.scala index c5b8c09..3ac2e7b 100644 --- core/src/main/scala/kafka/Kafka.scala +++ core/src/main/scala/kafka/Kafka.scala @@ -17,8 +17,6 @@ package kafka -import consumer.ConsumerConfig -import producer.ProducerConfig import server.{KafkaConfig, KafkaServerStartable, KafkaServer} import utils.{Utils, Logging} import org.apache.log4j.jmx.LoggerDynamicMBean @@ -30,8 +28,8 @@ object Kafka extends Logging { import org.apache.log4j.Logger Utils.registerMBean(new LoggerDynamicMBean(Logger.getRootLogger()), kafkaLog4jMBeanName) - if (!List(1, 3).contains(args.length)) { - println("USAGE: java [options] %s server.properties [consumer.properties producer.properties]".format(classOf[KafkaServer].getSimpleName())) + if (args.length != 1) { + println("USAGE: java [options] %s server.properties".format(classOf[KafkaServer].getSimpleName())) System.exit(1) } @@ -39,14 +37,7 @@ object Kafka extends Logging { val props = Utils.loadProps(args(0)) val serverConfig = new KafkaConfig(props) - val kafkaServerStartble = args.length match { - case 3 => - val consumerConfig = new ConsumerConfig(Utils.loadProps(args(1))) - val producerConfig = new ProducerConfig(Utils.loadProps(args(2))) - new KafkaServerStartable(serverConfig, consumerConfig, producerConfig) - case 1 => - new KafkaServerStartable(serverConfig) - } + val kafkaServerStartble = new KafkaServerStartable(serverConfig) // attach shutdown handler to catch control-c Runtime.getRuntime().addShutdownHook(new Thread() { diff --git core/src/main/scala/kafka/consumer/ConsoleConsumer.scala core/src/main/scala/kafka/consumer/ConsoleConsumer.scala index bddbb2b..49a6f39 100644 --- core/src/main/scala/kafka/consumer/ConsoleConsumer.scala +++ core/src/main/scala/kafka/consumer/ConsoleConsumer.scala @@ -36,11 +36,19 @@ object ConsoleConsumer extends Logging { def main(args: Array[String]) { val parser = new OptionParser - val topicIdOpt = parser.accepts("topic", "REQUIRED: The topic id to consume on.") + val topicIdOpt = parser.accepts("topic", "The topic id to consume on.") .withRequiredArg .describedAs("topic") .ofType(classOf[String]) - val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the form host:port. " + + val whitelistOpt = parser.accepts("whitelist", "Whitelist of topics to include for consumption.") + .withRequiredArg + .describedAs("whitelist") + .ofType(classOf[String]) + val blacklistOpt = parser.accepts("blacklist", "Blacklist of topics to exclude from consumption.") + .withRequiredArg + .describedAs("blacklist") + .ofType(classOf[String]) + val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the form host:port. " + "Multiple URLS can be given to allow fail-over.") .withRequiredArg .describedAs("urls") @@ -90,8 +98,20 @@ object ConsoleConsumer extends Logging { "skip it instead of halt.") val options: OptionSet = tryParse(parser, args) - checkRequiredArgs(parser, options, topicIdOpt, zkConnectOpt) + Utils.checkRequiredArgs(parser, options, zkConnectOpt) + val topicOrFilterOpt = List(topicIdOpt, whitelistOpt, blacklistOpt).filter(options.has) + if (topicOrFilterOpt.size != 1) { + error("Exactly one of whitelist/blacklist/topic is required.") + parser.printHelpOn(System.err) + System.exit(1) + } + val topicArg = options.valueOf(topicOrFilterOpt.head) + val filterSpec = if (options.has(blacklistOpt)) + new Blacklist(topicArg) + else + new Whitelist(topicArg) + val props = new Properties() props.put("groupid", options.valueOf(groupIdOpt)) props.put("socket.buffersize", options.valueOf(socketBufferSizeOpt).toString) @@ -104,7 +124,6 @@ object ConsoleConsumer extends Logging { val config = new ConsumerConfig(props) val skipMessageOnError = if (options.has(skipMessageOnErrorOpt)) true else false - val topic = options.valueOf(topicIdOpt) val messageFormatterClass = Class.forName(options.valueOf(messageFormatterOpt)) val formatterArgs = tryParseFormatterArgs(options.valuesOf(messageFormatterArgOpt)) @@ -123,21 +142,20 @@ object ConsoleConsumer extends Logging { tryCleanupZookeeper(options.valueOf(zkConnectOpt), options.valueOf(groupIdOpt)) } }) - - var stream = connector.createMessageStreams(Map(topic -> 1)).get(topic).get.get(0) - val iter = - if(maxMessages >= 0) - stream.slice(0, maxMessages) - else - stream + + val stream = connector.createMessageStreamsByFilter(filterSpec).get(0) + val iter = if(maxMessages >= 0) + stream.slice(0, maxMessages) + else + stream val formatter: MessageFormatter = messageFormatterClass.newInstance().asInstanceOf[MessageFormatter] formatter.init(formatterArgs) try { - for(message <- iter) { + for(messageAndTopic <- iter) { try { - formatter.writeTo(message, System.out) + formatter.writeTo(messageAndTopic.message, System.out) } catch { case e => if (skipMessageOnError) @@ -173,16 +191,6 @@ object ConsoleConsumer extends Logging { } } - def checkRequiredArgs(parser: OptionParser, options: OptionSet, required: OptionSpec[_]*) { - for(arg <- required) { - if(!options.has(arg)) { - error("Missing required argument \"" + arg + "\"") - parser.printHelpOn(System.err) - System.exit(1) - } - } - } - def tryParseFormatterArgs(args: Iterable[String]): Properties = { val splits = args.map(_ split "=").filterNot(_ == null).filterNot(_.length == 0) if(!splits.forall(_.length == 2)) { diff --git core/src/main/scala/kafka/consumer/ConsumerConfig.scala core/src/main/scala/kafka/consumer/ConsumerConfig.scala index 9cfda55..c531cd1 100644 --- core/src/main/scala/kafka/consumer/ConsumerConfig.scala +++ core/src/main/scala/kafka/consumer/ConsumerConfig.scala @@ -20,7 +20,6 @@ package kafka.consumer import java.util.Properties import kafka.utils.{ZKConfig, Utils} import kafka.api.OffsetRequest -import kafka.common.InvalidConfigException object ConsumerConfig { val SocketTimeout = 30 * 1000 val SocketBufferSize = 64*1024 @@ -90,27 +89,10 @@ class ConsumerConfig(props: Properties) extends ZKConfig(props) { /** throw a timeout exception to the consumer if no message is available for consumption after the specified interval */ val consumerTimeoutMs = Utils.getInt(props, "consumer.timeout.ms", ConsumerTimeoutMs) - /** Whitelist of topics for this mirror's embedded consumer to consume. At - * most one of whitelist/blacklist may be specified. */ - val mirrorTopicsWhitelist = Utils.getString( - props, MirrorTopicsWhitelistProp, MirrorTopicsWhitelist) - - /** Topics to skip mirroring. At most one of whitelist/blacklist may be - * specified */ - val mirrorTopicsBlackList = Utils.getString( - props, MirrorTopicsBlacklistProp, MirrorTopicsBlacklist) - - if (mirrorTopicsWhitelist.nonEmpty && mirrorTopicsBlackList.nonEmpty) - throw new InvalidConfigException("The embedded consumer's mirror topics configuration can only contain one of blacklist or whitelist") - - val mirrorConsumerNumThreads = Utils.getInt( - props, MirrorConsumerNumThreadsProp, MirrorConsumerNumThreads) - /** Use shallow iterator over compressed messages directly. This feature should be used very carefully. * Typically, it's only used for mirroring raw messages from one kafka cluster to another to save the * overhead of decompression. * */ val enableShallowIterator = Utils.getBoolean(props, "shallowiterator.enable", false) - } diff --git core/src/main/scala/kafka/consumer/ConsumerConnector.scala core/src/main/scala/kafka/consumer/ConsumerConnector.scala index 14671d6..94cb2f1 100644 --- core/src/main/scala/kafka/consumer/ConsumerConnector.scala +++ core/src/main/scala/kafka/consumer/ConsumerConnector.scala @@ -29,12 +29,28 @@ trait ConsumerConnector { * Create a list of MessageStreams for each topic. * * @param topicCountMap a map of (topic, #streams) pair - * @return a map of (topic, list of KafkaMessageStream) pair. The number of items in the - * list is #streams. Each KafkaMessageStream supports an iterator of messages. + * @param decoder Decoder to decode each Message to type T + * @return a map of (topic, list of KafkaStream) pairs. + * The number of items in the list is #streams. Each stream supports + * an iterator over message/metadata pairs. */ def createMessageStreams[T](topicCountMap: Map[String,Int], decoder: Decoder[T] = new DefaultDecoder) - : Map[String,List[KafkaMessageStream[T]]] + : Map[String,List[KafkaStream[T]]] + + /** + * Create a list of message streams for all topics that match a given filter. + * + * @param topicFilter Either a Whitelist or Blacklist TopicFilter object. + * @param numStreams Number of streams to return + * @param decoder Decoder to decode each Message to type T + * @return a list of KafkaStream each of which provides an + * iterator over message/metadata pairs over allowed topics. + */ + def createMessageStreamsByFilter[T](topicFilter: TopicFilter, + numStreams: Int = 1, + decoder: Decoder[T] = new DefaultDecoder) + : Seq[KafkaStream[T]] /** * Commit the offsets of all broker partitions connected by this connector. diff --git core/src/main/scala/kafka/consumer/ConsumerIterator.scala core/src/main/scala/kafka/consumer/ConsumerIterator.scala index e8bc4f9..9c6828c 100644 --- core/src/main/scala/kafka/consumer/ConsumerIterator.scala +++ core/src/main/scala/kafka/consumer/ConsumerIterator.scala @@ -19,37 +19,38 @@ package kafka.consumer import kafka.utils.{IteratorTemplate, Logging} import java.util.concurrent.{TimeUnit, BlockingQueue} -import kafka.message.MessageAndOffset import kafka.serializer.Decoder import java.util.concurrent.atomic.AtomicReference +import kafka.message.{MessageAndOffset, MessageAndMetadata} + /** * An iterator that blocks until a value can be read from the supplied queue. * The iterator takes a shutdownCommand object which can be added to the queue to trigger a shutdown * */ -class ConsumerIterator[T](private val topic: String, - private val channel: BlockingQueue[FetchedDataChunk], +class ConsumerIterator[T](private val channel: BlockingQueue[FetchedDataChunk], consumerTimeoutMs: Int, private val decoder: Decoder[T], val enableShallowIterator: Boolean) - extends IteratorTemplate[T] with Logging { + extends IteratorTemplate[MessageAndMetadata[T]] with Logging { private var current: AtomicReference[Iterator[MessageAndOffset]] = new AtomicReference(null) private var currentTopicInfo:PartitionTopicInfo = null private var consumedOffset: Long = -1L - override def next(): T = { - val decodedMessage = super.next() + override def next(): MessageAndMetadata[T] = { + val item = super.next() if(consumedOffset < 0) throw new IllegalStateException("Offset returned by the message set is invalid %d".format(consumedOffset)) currentTopicInfo.resetConsumeOffset(consumedOffset) - trace("Setting consumed offset to %d".format(consumedOffset)) + val topic = currentTopicInfo.topic + trace("Setting %s consumed offset to %d".format(topic, consumedOffset)) ConsumerTopicStat.getConsumerTopicStat(topic).recordMessagesPerTopic(1) - decodedMessage + item } - protected def makeNext(): T = { + protected def makeNext(): MessageAndMetadata[T] = { var currentDataChunk: FetchedDataChunk = null // if we don't have an iterator, get one var localCurrent = current.get() @@ -82,10 +83,11 @@ class ConsumerIterator[T](private val topic: String, } val item = localCurrent.next() consumedOffset = item.offset - decoder.toEvent(item.message) + + new MessageAndMetadata(decoder.toEvent(item.message), currentTopicInfo.topic) } - def clearCurrentChunk() = { + def clearCurrentChunk() { try { info("Clearing the current data chunk for this consumer iterator") current.set(null) @@ -94,3 +96,4 @@ class ConsumerIterator[T](private val topic: String, } class ConsumerTimeoutException() extends RuntimeException() + diff --git core/src/main/scala/kafka/consumer/Fetcher.scala core/src/main/scala/kafka/consumer/Fetcher.scala index d18faca..5e65df9 100644 --- core/src/main/scala/kafka/consumer/Fetcher.scala +++ core/src/main/scala/kafka/consumer/Fetcher.scala @@ -42,24 +42,24 @@ private [consumer] class Fetcher(val config: ConsumerConfig, val zkClient : ZkCl fetcherThreads = EMPTY_FETCHER_THREADS } - def clearFetcherQueues[T](topicInfos: Iterable[PartitionTopicInfo], cluster: Cluster, + def clearFetcherQueues(topicInfos: Iterable[PartitionTopicInfo], cluster: Cluster, queuesTobeCleared: Iterable[BlockingQueue[FetchedDataChunk]], - kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]]) { + messageStreams: Map[String,List[KafkaStream[_]]]) { // Clear all but the currently iterated upon chunk in the consumer thread's queue queuesTobeCleared.foreach(_.clear) info("Cleared all relevant queues for this fetcher") // Also clear the currently iterated upon chunk in the consumer threads - if(kafkaMessageStreams != null) - kafkaMessageStreams.foreach(_._2.foreach(s => s.clear())) + if(messageStreams != null) + messageStreams.foreach(_._2.foreach(s => s.clear())) info("Cleared the data chunks in all the consumer message iterators") } - def startConnections[T](topicInfos: Iterable[PartitionTopicInfo], cluster: Cluster, - kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]]) { + def startConnections(topicInfos: Iterable[PartitionTopicInfo], + cluster: Cluster) { if (topicInfos == null) return diff --git core/src/main/scala/kafka/consumer/KafkaMessageStream.scala core/src/main/scala/kafka/consumer/KafkaMessageStream.scala deleted file mode 100644 index 288d859..0000000 --- core/src/main/scala/kafka/consumer/KafkaMessageStream.scala +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.consumer - -import java.util.concurrent.BlockingQueue -import kafka.serializer.Decoder - -/** - * All calls to elements should produce the same thread-safe iterator? Should have a separate thread - * that feeds messages into a blocking queue for processing. - */ -class KafkaMessageStream[T](val topic: String, - private val queue: BlockingQueue[FetchedDataChunk], - consumerTimeoutMs: Int, - private val decoder: Decoder[T], - val enableShallowIterator: Boolean) - extends Iterable[T] with java.lang.Iterable[T]{ - - private val iter: ConsumerIterator[T] = - new ConsumerIterator[T](topic, queue, consumerTimeoutMs, decoder, enableShallowIterator) - - /** - * Create an iterator over messages in the stream. - */ - def iterator(): ConsumerIterator[T] = iter - - /** - * This method clears the queue being iterated during the consumer rebalancing. This is mainly - * to reduce the number of duplicates received by the consumer - */ - def clear() { - iter.clearCurrentChunk() - } - -} diff --git core/src/main/scala/kafka/consumer/KafkaStream.scala core/src/main/scala/kafka/consumer/KafkaStream.scala new file mode 100644 index 0000000..3ef0978 --- /dev/null +++ core/src/main/scala/kafka/consumer/KafkaStream.scala @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.consumer + + +import java.util.concurrent.BlockingQueue +import kafka.serializer.Decoder +import kafka.message.MessageAndMetadata + +class KafkaStream[T](private val queue: BlockingQueue[FetchedDataChunk], + consumerTimeoutMs: Int, + private val decoder: Decoder[T], + val enableShallowIterator: Boolean) + extends Iterable[MessageAndMetadata[T]] with java.lang.Iterable[MessageAndMetadata[T]] { + + private val iter: ConsumerIterator[T] = + new ConsumerIterator[T](queue, consumerTimeoutMs, decoder, enableShallowIterator) + + /** + * Create an iterator over messages in the stream. + */ + def iterator(): ConsumerIterator[T] = iter + + /** + * This method clears the queue being iterated during the consumer rebalancing. This is mainly + * to reduce the number of duplicates received by the consumer + */ + def clear() { + iter.clearCurrentChunk() + } + +} diff --git core/src/main/scala/kafka/consumer/TopicCount.scala core/src/main/scala/kafka/consumer/TopicCount.scala index 51bf516..c60f5a3 100644 --- core/src/main/scala/kafka/consumer/TopicCount.scala +++ core/src/main/scala/kafka/consumer/TopicCount.scala @@ -19,35 +19,18 @@ package kafka.consumer import scala.collection._ import scala.util.parsing.json.JSON -import kafka.utils.Logging +import org.I0Itec.zkclient.ZkClient +import java.util.regex.Pattern +import kafka.utils.{ZKGroupDirs, ZkUtils, Logging} -private[kafka] object TopicCount extends Logging { - val myConversionFunc = {input : String => input.toInt} - JSON.globalNumberParser = myConversionFunc - - def constructTopicCount(consumerIdSting: String, jsonString : String) : TopicCount = { - var topMap : Map[String,Int] = null - try { - JSON.parseFull(jsonString) match { - case Some(m) => topMap = m.asInstanceOf[Map[String,Int]] - case None => throw new RuntimeException("error constructing TopicCount : " + jsonString) - } - } - catch { - case e => - error("error parsing consumer json string " + jsonString, e) - throw e - } - - new TopicCount(consumerIdSting, topMap) - } -} - -private[kafka] class TopicCount(val consumerIdString: String, val topicCountMap: Map[String, Int]) { +private[kafka] trait TopicCount { + def getConsumerThreadIdsPerTopic: Map[String, Set[String]] - def getConsumerThreadIdsPerTopic() - : Map[String, Set[String]] = { + def dbString: String + + protected def makeConsumerThreadIdsPerTopic(consumerIdString: String, + topicCountMap: Map[String, Int]) = { val consumerThreadIdsPerTopicMap = new mutable.HashMap[String, Set[String]]() for ((topic, nConsumers) <- topicCountMap) { val consumerSet = new mutable.HashSet[String] @@ -58,11 +41,96 @@ private[kafka] class TopicCount(val consumerIdString: String, val topicCountMap: } consumerThreadIdsPerTopicMap } +} + +private[kafka] object TopicCount extends Logging { + + /* + * Example of whitelist topic count stored in ZooKeeper: + * Topics with whitetopic as prefix, and four streams: *4*whitetopic.* + * + * Example of blacklist topic count stored in ZooKeeper: + * Topics with blacktopic as prefix, and four streams: !4!blacktopic.* + */ + + val WHITELIST_MARKER = "*" + val BLACKLIST_MARKER = "!" + private val WHITELIST_PATTERN = + Pattern.compile("""\*(\p{Digit}+)\*(.*)""") + private val BLACKLIST_PATTERN = + Pattern.compile("""!(\p{Digit}+)!(.*)""") + + val myConversionFunc = {input : String => input.toInt} + JSON.globalNumberParser = myConversionFunc + + def constructTopicCount(group: String, + consumerId: String, + zkClient: ZkClient) : TopicCount = { + val dirs = new ZKGroupDirs(group) + val topicCountString = ZkUtils.readData(zkClient, dirs.consumerRegistryDir + "/" + consumerId) + val hasWhitelist = topicCountString.startsWith(WHITELIST_MARKER) + val hasBlacklist = topicCountString.startsWith(BLACKLIST_MARKER) + + if (hasWhitelist || hasBlacklist) + info("Constructing topic count for %s from %s using %s as pattern." + .format(consumerId, topicCountString, + if (hasWhitelist) WHITELIST_PATTERN else BLACKLIST_PATTERN)) + + if (hasWhitelist || hasBlacklist) { + val matcher = if (hasWhitelist) + WHITELIST_PATTERN.matcher(topicCountString) + else + BLACKLIST_PATTERN.matcher(topicCountString) + require(matcher.matches()) + val numStreams = matcher.group(1).toInt + val regex = matcher.group(2) + val filter = if (hasWhitelist) + new Whitelist(regex) + else + new Blacklist(regex) + + new WildcardTopicCount(zkClient, consumerId, filter, numStreams) + } + else { + var topMap : Map[String,Int] = null + try { + JSON.parseFull(topicCountString) match { + case Some(m) => topMap = m.asInstanceOf[Map[String,Int]] + case None => throw new RuntimeException("error constructing TopicCount : " + topicCountString) + } + } + catch { + case e => + error("error parsing consumer json string " + topicCountString, e) + throw e + } + + new StaticTopicCount(consumerId, topMap) + } + } + + def constructTopicCount(consumerIdString: String, topicCount: Map[String, Int]) = + new StaticTopicCount(consumerIdString, topicCount) + + def constructTopicCount(consumerIdString: String, + filter: TopicFilter, + numStreams: Int, + zkClient: ZkClient) = + new WildcardTopicCount(zkClient, consumerIdString, filter, numStreams) + +} + +private[kafka] class StaticTopicCount(val consumerIdString: String, + val topicCountMap: Map[String, Int]) + extends TopicCount { + + def getConsumerThreadIdsPerTopic = + makeConsumerThreadIdsPerTopic(consumerIdString, topicCountMap) override def equals(obj: Any): Boolean = { obj match { case null => false - case n: TopicCount => consumerIdString == n.consumerIdString && topicCountMap == n.topicCountMap + case n: StaticTopicCount => consumerIdString == n.consumerIdString && topicCountMap == n.topicCountMap case _ => false } } @@ -73,7 +141,7 @@ private[kafka] class TopicCount(val consumerIdString: String, val topicCountMap: * "topic2" : 4 * } */ - def toJsonString() : String = { + def dbString = { val builder = new StringBuilder builder.append("{ ") var i = 0 @@ -84,6 +152,29 @@ private[kafka] class TopicCount(val consumerIdString: String, val topicCountMap: i += 1 } builder.append(" }") - builder.toString + builder.toString() } } + +private[kafka] class WildcardTopicCount(zkClient: ZkClient, + consumerIdString: String, + topicFilter: TopicFilter, + numStreams: Int) extends TopicCount { + def getConsumerThreadIdsPerTopic = { + val wildcardTopics = ZkUtils.getChildrenParentMayNotExist( + zkClient, ZkUtils.BrokerTopicsPath).filter(topicFilter.isTopicAllowed(_)) + makeConsumerThreadIdsPerTopic(consumerIdString, + Map(wildcardTopics.map((_, numStreams)): _*)) + } + + def dbString = { + val marker = topicFilter match { + case wl: Whitelist => TopicCount.WHITELIST_MARKER + case bl: Blacklist => TopicCount.BLACKLIST_MARKER + } + + "%s%d%s%s".format(marker, numStreams, marker, topicFilter.regex) + } + +} + diff --git core/src/main/scala/kafka/consumer/TopicFilter.scala core/src/main/scala/kafka/consumer/TopicFilter.scala new file mode 100644 index 0000000..cf3853b --- /dev/null +++ core/src/main/scala/kafka/consumer/TopicFilter.scala @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.consumer + + +import kafka.utils.Logging +import java.util.regex.{PatternSyntaxException, Pattern} + + +sealed abstract class TopicFilter(rawRegex: String) extends Logging { + + val regex = rawRegex + .trim + .replace(',', '|') + .replace(" ", "") + .replaceAll("""^["']+""","") + .replaceAll("""["']+$""","") // property files may bring quotes + + try { + Pattern.compile(regex) + } + catch { + case e: PatternSyntaxException => + throw new RuntimeException(regex + " is an invalid regex.") + } + + override def toString = regex + + def requiresTopicEventWatcher: Boolean + + def isTopicAllowed(topic: String): Boolean +} + +case class Whitelist(rawRegex: String) extends TopicFilter(rawRegex) { + override def requiresTopicEventWatcher = !regex.matches("""[\p{Alnum}-|]+""") + + override def isTopicAllowed(topic: String) = { + val allowed = topic.matches(regex) + + debug("%s %s".format( + topic, if (allowed) "allowed" else "filtered")) + + allowed + } + + +} + +case class Blacklist(rawRegex: String) extends TopicFilter(rawRegex) { + override def requiresTopicEventWatcher = true + + override def isTopicAllowed(topic: String) = { + val allowed = !topic.matches(regex) + + debug("%s %s".format( + topic, if (allowed) "allowed" else "filtered")) + + allowed + } +} + diff --git core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 595efea..f7782df 100644 --- core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,6 +34,7 @@ import kafka.common.{ConsumerRebalanceFailedException, InvalidConfigException} import java.lang.IllegalStateException import kafka.utils.ZkUtils._ + /** * This class handles the consumers interaction with zookeeper * @@ -86,16 +87,37 @@ trait ZookeeperConsumerConnectorMBean { private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val enableFetcher: Boolean) // for testing only - extends ConsumerConnector with ZookeeperConsumerConnectorMBean with Logging { - + extends ConsumerConnector with ZookeeperConsumerConnectorMBean + with Logging { private val isShuttingDown = new AtomicBoolean(false) private val rebalanceLock = new Object private var fetcher: Option[Fetcher] = None private var zkClient: ZkClient = null private var topicRegistry = new Pool[String, Pool[Partition, PartitionTopicInfo]] - // queues : (topic,consumerThreadId) -> queue - private val queues = new Pool[Tuple2[String,String], BlockingQueue[FetchedDataChunk]] + // topicThreadIdAndQueues : (topic,consumerThreadId) -> queue + private val topicThreadIdAndQueues = new Pool[Tuple2[String,String], BlockingQueue[FetchedDataChunk]] private val scheduler = new KafkaScheduler(1, "Kafka-consumer-autocommit-", false) + private val messageStreamCreated = new AtomicBoolean(false) + + private var sessionExpirationListener: ZKSessionExpireListener = null + private var loadBalancerListener: ZKRebalancerListener = null + + private var wildcardTopicWatcher: ZookeeperTopicEventWatcher = null + + val consumerIdString = { + var consumerUuid : String = null + config.consumerId match { + case Some(consumerId) // for testing only + => consumerUuid = consumerId + case None // generate unique consumerId automatically + => val uuid = UUID.randomUUID() + consumerUuid = "%s-%d-%s".format( + InetAddress.getLocalHost.getHostName, System.currentTimeMillis, + uuid.getMostSignificantBits().toHexString.substring(0,8)) + } + config.groupId + "_" + consumerUuid + } + this.logIdent = consumerIdString + " " connectZk() createFetcher() @@ -108,10 +130,18 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, def createMessageStreams[T](topicCountMap: Map[String,Int], decoder: Decoder[T]) - : Map[String,List[KafkaMessageStream[T]]] = { + : Map[String,List[KafkaStream[T]]] = { + if (messageStreamCreated.getAndSet(true)) + throw new RuntimeException(this.getClass.getSimpleName + + " can create message streams at most once") consume(topicCountMap, decoder) } + def createMessageStreamsByFilter[T](topicFilter: TopicFilter, numStreams: Int, decoder: Decoder[T]) = { + val wildcardStreamsHandler = new WildcardStreamsHandler[T](topicFilter, numStreams, decoder) + wildcardStreamsHandler.streams + } + private def createFetcher() { if (enableFetcher) fetcher = Some(new Fetcher(config, zkClient)) @@ -126,6 +156,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val canShutdown = isShuttingDown.compareAndSet(false, true); if (canShutdown) { info("ZKConsumerConnector shutting down") + + if (wildcardTopicWatcher != null) + wildcardTopicWatcher.shutdown() try { scheduler.shutdownNow() fetcher match { @@ -150,69 +183,42 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, def consume[T](topicCountMap: scala.collection.Map[String,Int], decoder: Decoder[T]) - : Map[String,List[KafkaMessageStream[T]]] = { + : Map[String,List[KafkaStream[T]]] = { debug("entering consume ") if (topicCountMap == null) throw new RuntimeException("topicCountMap is null") - val dirs = new ZKGroupDirs(config.groupId) - var ret = new mutable.HashMap[String,List[KafkaMessageStream[T]]] + val topicCount = TopicCount.constructTopicCount(consumerIdString, topicCountMap) - var consumerUuid : String = null - config.consumerId match { - case Some(consumerId) // for testing only - => consumerUuid = consumerId - case None // generate unique consumerId automatically - => val uuid = UUID.randomUUID() - consumerUuid = "%s-%d-%s".format( - InetAddress.getLocalHost.getHostName, System.currentTimeMillis, - uuid.getMostSignificantBits().toHexString.substring(0,8)) - } - val consumerIdString = config.groupId + "_" + consumerUuid - val topicCount = new TopicCount(consumerIdString, topicCountMap) - - // create a queue per topic per consumer thread - val consumerThreadIdsPerTopic = topicCount.getConsumerThreadIdsPerTopic - for ((topic, threadIdSet) <- consumerThreadIdsPerTopic) { - var streamList: List[KafkaMessageStream[T]] = Nil - for (threadId <- threadIdSet) { - val stream = new LinkedBlockingQueue[FetchedDataChunk](config.maxQueuedChunks) - queues.put((topic, threadId), stream) - streamList ::= new KafkaMessageStream[T](topic, stream, config.consumerTimeoutMs, decoder, config.enableShallowIterator) - } - ret += (topic -> streamList) - debug("adding topic " + topic + " and stream to map..") - } - - // listener to consumer and partition changes - val loadBalancerListener = new ZKRebalancerListener[T](config.groupId, consumerIdString, ret) - registerConsumerInZK(dirs, consumerIdString, topicCount) + val topicThreadIds = topicCount.getConsumerThreadIdsPerTopic - // register listener for session expired event - zkClient.subscribeStateChanges( - new ZKSessionExpireListener[T](dirs, consumerIdString, topicCount, loadBalancerListener)) + // make a list of (queue,stream) pairs, one pair for each threadId + val queuesAndStreams = topicThreadIds.values.map(threadIdSet => + threadIdSet.map(_ => { + val queue = new LinkedBlockingQueue[FetchedDataChunk](config.maxQueuedChunks) + val stream = new KafkaStream[T]( + queue, config.consumerTimeoutMs, decoder, config.enableShallowIterator) + (queue, stream) + }) + ).flatten.toList - zkClient.subscribeChildChanges(dirs.consumerRegistryDir, loadBalancerListener) - - ret.foreach { topicAndStreams => - // register on broker partition path changes - val partitionPath = BrokerTopicsPath + "/" + topicAndStreams._1 - zkClient.subscribeChildChanges(partitionPath, loadBalancerListener) - } + val dirs = new ZKGroupDirs(config.groupId) + registerConsumerInZK(dirs, consumerIdString, topicCount) + reinitializeConsumer(topicCount, queuesAndStreams) - // explicitly trigger load balancing for this consumer - loadBalancerListener.syncedRebalance() - ret + loadBalancerListener.kafkaMessageAndMetadataStreams.asInstanceOf[Map[String, List[KafkaStream[T]]]] } - private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) = { + private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") - createEphemeralPathExpectConflict(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, topicCount.toJsonString) + createEphemeralPathExpectConflict(zkClient, + dirs.consumerRegistryDir + "/" + consumerIdString, + topicCount.dbString) info("end registering consumer " + consumerIdString + " in ZK") } private def sendShutdownToAllQueues() = { - for (queue <- queues.values) { + for (queue <- topicThreadIdAndQueues.values) { debug("Clearing up queue") queue.clear() queue.put(ZookeeperConsumerConnector.shutdownCommand) @@ -334,10 +340,10 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, producedOffset } - class ZKSessionExpireListener[T](val dirs: ZKGroupDirs, + class ZKSessionExpireListener(val dirs: ZKGroupDirs, val consumerIdString: String, val topicCount: TopicCount, - val loadBalancerListener: ZKRebalancerListener[T]) + val loadBalancerListener: ZKRebalancerListener) extends IZkStateListener { @throws(classOf[Exception]) def handleStateChanged(state: KeeperState) { @@ -359,10 +365,10 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, * consumer in the consumer registry and trigger a rebalance. */ info("ZK expired; release old broker parition ownership; re-register consumer " + consumerIdString) - loadBalancerListener.resetState + loadBalancerListener.resetState() registerConsumerInZK(dirs, consumerIdString, topicCount) // explicitly trigger load balancing for this consumer - loadBalancerListener.syncedRebalance + loadBalancerListener.syncedRebalance() // There is no need to resubscribe to child and state changes. // The child change watchers will be set inside rebalance when we read the children list. @@ -370,8 +376,8 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, } - class ZKRebalancerListener[T](val group: String, val consumerIdString: String, - kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]]) + class ZKRebalancerListener(val group: String, val consumerIdString: String, + val kafkaMessageAndMetadataStreams: mutable.Map[String,List[KafkaStream[_]]]) extends IZkChildListener { private var isWatcherTriggered = false private val lock = new ReentrantLock @@ -459,7 +465,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, info("Rebalancing attempt failed. Clearing the cache before the next rebalancing operation is triggered") } // stop all fetchers and clear all the queues to avoid data duplication - closeFetchersForQueues(cluster, kafkaMessageStreams, queues.map(q => q._2)) + closeFetchersForQueues(cluster, kafkaMessageAndMetadataStreams, topicThreadIdAndQueues.map(q => q._2)) Thread.sleep(config.rebalanceBackoffMs) } } @@ -468,7 +474,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, } private def rebalance(cluster: Cluster): Boolean = { - val myTopicThreadIdsMap = getTopicCount(zkClient, group, consumerIdString).getConsumerThreadIdsPerTopic + val myTopicThreadIdsMap = TopicCount.constructTopicCount(group, consumerIdString, zkClient).getConsumerThreadIdsPerTopic val consumersPerTopicMap = getConsumersPerTopic(zkClient, group) val partitionsPerTopicMap = getPartitionsForTopics(zkClient, myTopicThreadIdsMap.keys.iterator) @@ -478,7 +484,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, * But if we don't stop the fetchers first, this consumer would continue returning data for released * partitions in parallel. So, not stopping the fetchers leads to duplicate data. */ - closeFetchers(cluster, kafkaMessageStreams, myTopicThreadIdsMap) + closeFetchers(cluster, kafkaMessageAndMetadataStreams, myTopicThreadIdsMap) releasePartitionOwnership(topicRegistry) @@ -531,7 +537,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, debug("Partitions per topic cache " + partitionsPerTopicMap) debug("Consumers per topic cache " + consumersPerTopicMap) topicRegistry = currentTopicRegistry - updateFetcher(cluster, kafkaMessageStreams) + updateFetcher(cluster) true }else { false @@ -539,12 +545,12 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, } private def closeFetchersForQueues(cluster: Cluster, - kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]], + messageStreams: Map[String,List[KafkaStream[_]]], queuesToBeCleared: Iterable[BlockingQueue[FetchedDataChunk]]) { var allPartitionInfos = topicRegistry.values.map(p => p.values).flatten fetcher match { case Some(f) => f.stopConnectionsToAllBrokers - f.clearFetcherQueues(allPartitionInfos, cluster, queuesToBeCleared, kafkaMessageStreams) + f.clearFetcherQueues(allPartitionInfos, cluster, queuesToBeCleared, messageStreams) info("Committing all offsets after clearing the fetcher queues") /** * here, we need to commit offsets before stopping the consumer from returning any more messages @@ -559,16 +565,15 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, } } - private def closeFetchers(cluster: Cluster, kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]], + private def closeFetchers(cluster: Cluster, messageStreams: Map[String,List[KafkaStream[_]]], relevantTopicThreadIdsMap: Map[String, Set[String]]) { // only clear the fetcher queues for certain topic partitions that *might* no longer be served by this consumer // after this rebalancing attempt - val queuesTobeCleared = queues.filter(q => relevantTopicThreadIdsMap.contains(q._1._1)).map(q => q._2) - closeFetchersForQueues(cluster, kafkaMessageStreams, queuesTobeCleared) + val queuesTobeCleared = topicThreadIdAndQueues.filter(q => relevantTopicThreadIdsMap.contains(q._1._1)).map(q => q._2) + closeFetchersForQueues(cluster, messageStreams, queuesTobeCleared) } - private def updateFetcher[T](cluster: Cluster, - kafkaMessageStreams: Map[String,List[KafkaMessageStream[T]]]) { + private def updateFetcher(cluster: Cluster) { // update partitions for fetcher var allPartitionInfos : List[PartitionTopicInfo] = Nil for (partitionInfos <- topicRegistry.values) @@ -579,7 +584,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, fetcher match { case Some(f) => - f.startConnections(allPartitionInfos, cluster, kafkaMessageStreams) + f.startConnections(allPartitionInfos, cluster) case None => } } @@ -637,7 +642,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, } else offset = offsetString.toLong - val queue = queues.get((topic, consumerThreadId)) + val queue = topicThreadIdAndQueues.get((topic, consumerThreadId)) val consumedOffset = new AtomicLong(offset) val fetchedOffset = new AtomicLong(offset) val partTopicInfo = new PartitionTopicInfo(topic, @@ -651,5 +656,155 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, debug(partTopicInfo + " selected new offset " + offset) } } + + private def reinitializeConsumer[T]( + topicCount: TopicCount, + queuesAndStreams: List[(LinkedBlockingQueue[FetchedDataChunk],KafkaStream[T])]) { + + val dirs = new ZKGroupDirs(config.groupId) + + // listener to consumer and partition changes + if (loadBalancerListener == null) { + val topicStreamsMap = new mutable.HashMap[String,List[KafkaStream[T]]] + loadBalancerListener = new ZKRebalancerListener( + config.groupId, consumerIdString, topicStreamsMap.asInstanceOf[scala.collection.mutable.Map[String, List[KafkaStream[_]]]]) + } + + // register listener for session expired event + if (sessionExpirationListener == null) + sessionExpirationListener = new ZKSessionExpireListener( + dirs, consumerIdString, topicCount, loadBalancerListener) + + val topicStreamsMap = loadBalancerListener.kafkaMessageAndMetadataStreams + + // map of {topic -> Set(thread-1, thread-2, ...)} + val consumerThreadIdsPerTopic: Map[String, Set[String]] = + topicCount.getConsumerThreadIdsPerTopic + + /* + * This usage of map flatten breaks up consumerThreadIdsPerTopic into + * a set of (topic, thread-id) pairs that we then use to construct + * the updated (topic, thread-id) -> queues map + */ + implicit def getTopicThreadIds(v: (String, Set[String])): Set[(String, String)] = v._2.map((v._1, _)) + + // iterator over (topic, thread-id) tuples + val topicThreadIds: Iterable[(String, String)] = + consumerThreadIdsPerTopic.flatten + + // list of (pairs of pairs): e.g., ((topic, thread-id),(queue, stream)) + val threadQueueStreamPairs = topicCount match { + case wildTopicCount: WildcardTopicCount => + for (tt <- topicThreadIds; qs <- queuesAndStreams) yield (tt -> qs) + case statTopicCount: StaticTopicCount => { + require(topicThreadIds.size == queuesAndStreams.size, + "Mismatch between thread ID count (%d) and queue count (%d)".format( + topicThreadIds.size, queuesAndStreams.size)) + topicThreadIds.zip(queuesAndStreams) + } + } + + threadQueueStreamPairs.foreach(e => { + val topicThreadId = e._1 + val q = e._2._1 + topicThreadIdAndQueues.put(topicThreadId, q) + }) + + val groupedByTopic = threadQueueStreamPairs.groupBy(_._1._1) + groupedByTopic.foreach(e => { + val topic = e._1 + val streams = e._2.map(_._2._2).toList + topicStreamsMap += (topic -> streams) + debug("adding topic %s and %d streams to map.".format(topic, streams.size)) + }) + + // listener to consumer and partition changes + zkClient.subscribeStateChanges(sessionExpirationListener) + + zkClient.subscribeChildChanges(dirs.consumerRegistryDir, loadBalancerListener) + + topicStreamsMap.foreach { topicAndStreams => + // register on broker partition path changes + val partitionPath = BrokerTopicsPath + "/" + topicAndStreams._1 + zkClient.subscribeChildChanges(partitionPath, loadBalancerListener) + } + + // explicitly trigger load balancing for this consumer + loadBalancerListener.syncedRebalance() + } + + class WildcardStreamsHandler[T](topicFilter: TopicFilter, + numStreams: Int, + decoder: Decoder[T]) + extends TopicEventHandler[String] { + + if (messageStreamCreated.getAndSet(true)) + throw new RuntimeException("Each consumer connector can create " + + "message streams by filter at most once.") + + private val wildcardQueuesAndStreams = (1 to numStreams) + .map(e => { + val queue = new LinkedBlockingQueue[FetchedDataChunk](config.maxQueuedChunks) + val stream = new KafkaStream[T]( + queue, config.consumerTimeoutMs, decoder, config.enableShallowIterator) + (queue, stream) + }).toList + + // bootstrap with existing topics + private var wildcardTopics = + getChildrenParentMayNotExist(zkClient, BrokerTopicsPath) + .filter(topicFilter.isTopicAllowed) + + private val wildcardTopicCount = TopicCount.constructTopicCount( + consumerIdString, topicFilter, numStreams, zkClient) + + val dirs = new ZKGroupDirs(config.groupId) + registerConsumerInZK(dirs, consumerIdString, wildcardTopicCount) + reinitializeConsumer(wildcardTopicCount, wildcardQueuesAndStreams) + + if (!topicFilter.requiresTopicEventWatcher) { + info("Not creating event watcher for trivial whitelist " + topicFilter) + } + else { + info("Creating topic event watcher for whitelist " + topicFilter) + wildcardTopicWatcher = new ZookeeperTopicEventWatcher(config, this) + + /* + * Topic events will trigger subsequent synced rebalances. Also, the + * consumer will get registered only after an allowed topic becomes + * available. + */ + } + + def handleTopicEvent(allTopics: Seq[String]) { + debug("Handling topic event") + + val updatedTopics = allTopics.filter(topicFilter.isTopicAllowed) + + val addedTopics = updatedTopics filterNot (wildcardTopics contains) + if (addedTopics.nonEmpty) + info("Topic event: added topics = %s" + .format(addedTopics)) + + /* + * TODO: Deleted topics are interesting (and will not be a concern until + * 0.8 release). We may need to remove these topics from the rebalance + * listener's map in reinitializeConsumer. + */ + val deletedTopics = wildcardTopics filterNot (updatedTopics contains) + if (deletedTopics.nonEmpty) + info("Topic event: deleted topics = %s" + .format(deletedTopics)) + + wildcardTopics = updatedTopics + info("Topics to consume = %s".format(wildcardTopics)) + + if (addedTopics.nonEmpty || deletedTopics.nonEmpty) + reinitializeConsumer(wildcardTopicCount, wildcardQueuesAndStreams) + } + + def streams: Seq[KafkaStream[T]] = + wildcardQueuesAndStreams.map(_._2) + } } diff --git core/src/main/scala/kafka/consumer/ZookeeperTopicEventWatcher.scala core/src/main/scala/kafka/consumer/ZookeeperTopicEventWatcher.scala index eb563e1..df83baa 100644 --- core/src/main/scala/kafka/consumer/ZookeeperTopicEventWatcher.scala +++ core/src/main/scala/kafka/consumer/ZookeeperTopicEventWatcher.scala @@ -21,11 +21,9 @@ import scala.collection.JavaConversions._ import kafka.utils.{ZkUtils, ZKStringSerializer, Logging} import org.I0Itec.zkclient.{IZkStateListener, IZkChildListener, ZkClient} import org.apache.zookeeper.Watcher.Event.KeeperState -import kafka.server.KafkaServerStartable -import kafka.common.ConsumerRebalanceFailedException class ZookeeperTopicEventWatcher(val config:ConsumerConfig, - val eventHandler: TopicEventHandler[String], kafkaServerStartable: KafkaServerStartable) extends Logging { + val eventHandler: TopicEventHandler[String]) extends Logging { val lock = new Object() @@ -35,7 +33,7 @@ class ZookeeperTopicEventWatcher(val config:ConsumerConfig, startWatchingTopicEvents() private def startWatchingTopicEvents() { - val topicEventListener = new ZkTopicEventListener(kafkaServerStartable) + val topicEventListener = new ZkTopicEventListener() ZkUtils.makeSurePersistentPathExists(zkClient, ZkUtils.BrokerTopicsPath) zkClient.subscribeStateChanges( @@ -52,6 +50,7 @@ class ZookeeperTopicEventWatcher(val config:ConsumerConfig, def shutdown() { lock.synchronized { + info("Shutting down topic event watcher.") if (zkClient != null) { stopWatchingTopicEvents() zkClient.close() @@ -62,7 +61,7 @@ class ZookeeperTopicEventWatcher(val config:ConsumerConfig, } } - class ZkTopicEventListener(val kafkaServerStartable: KafkaServerStartable) extends IZkChildListener { + class ZkTopicEventListener extends IZkChildListener { @throws(classOf[Exception]) def handleChildChange(parent: String, children: java.util.List[String]) { @@ -76,11 +75,8 @@ class ZookeeperTopicEventWatcher(val config:ConsumerConfig, } } catch { - case e: ConsumerRebalanceFailedException => - fatal("can't rebalance in embedded consumer; proceed to shutdown", e) - kafkaServerStartable.shutdown() case e => - error("error in handling child changes in embedded consumer", e) + error("error in handling child changes", e) } } } diff --git core/src/main/scala/kafka/javaapi/consumer/ConsumerConnector.java core/src/main/scala/kafka/javaapi/consumer/ConsumerConnector.java index 6f50e97..afb6b0a 100644 --- core/src/main/scala/kafka/javaapi/consumer/ConsumerConnector.java +++ core/src/main/scala/kafka/javaapi/consumer/ConsumerConnector.java @@ -17,34 +17,53 @@ package kafka.javaapi.consumer; -import kafka.consumer.KafkaMessageStream; -import kafka.message.Message; -import kafka.serializer.Decoder; import java.util.List; import java.util.Map; +import kafka.consumer.KafkaStream; +import kafka.consumer.TopicFilter; +import kafka.message.Message; +import kafka.serializer.Decoder; public interface ConsumerConnector { - /** - * Create a list of MessageStreams of type T for each topic. - * - * @param topicCountMap a map of (topic, #streams) pair - * @param decoder a decoder that converts from Message to T - * @return a map of (topic, list of KafkaMessageStream) pair. The number of items in the - * list is #streams. Each KafkaMessageStream supports an iterator of messages. - */ - public Map>> createMessageStreams( - Map topicCountMap, Decoder decoder); - public Map>> createMessageStreams( - Map topicCountMap); + /** + * Create a list of MessageStreams of type T for each topic. + * + * @param topicCountMap a map of (topic, #streams) pair + * @param decoder a decoder that converts from Message to T + * @return a map of (topic, list of KafkaStream) pairs. + * The number of items in the list is #streams. Each stream supports + * an iterator over message/metadata pairs. + */ + public Map>> createMessageStreams( + Map topicCountMap, Decoder decoder); + public Map>> createMessageStreams( + Map topicCountMap); + + /** + * Create a list of MessageAndTopicStreams containing messages of type T. + * + * @param topicFilter a TopicFilter that specifies which topics to + * subscribe to (encapsulates a whitelist or a blacklist). + * @param numStreams the number of message streams to return. + * @param decoder a decoder that converts from Message to T + * @return a list of KafkaStream. Each stream supports an + * iterator over its MessageAndMetadata elements. + */ + public List> createMessageStreamsByFilter( + TopicFilter topicFilter, int numStreams, Decoder decoder); + public List> createMessageStreamsByFilter( + TopicFilter topicFilter, int numStreams); + public List> createMessageStreamsByFilter( + TopicFilter topicFilter); - /** - * Commit the offsets of all broker partitions connected by this connector. - */ - public void commitOffsets(); + /** + * Commit the offsets of all broker partitions connected by this connector. + */ + public void commitOffsets(); - /** - * Shut down the connector - */ - public void shutdown(); + /** + * Shut down the connector + */ + public void shutdown(); } diff --git core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala index 0ee7488..f1a469b 100644 --- core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala +++ core/src/main/scala/kafka/javaapi/consumer/ZookeeperConsumerConnector.scala @@ -16,9 +16,11 @@ */ package kafka.javaapi.consumer -import kafka.consumer.{KafkaMessageStream, ConsumerConfig} import kafka.message.Message import kafka.serializer.{DefaultDecoder, Decoder} +import kafka.consumer._ +import scala.collection.JavaConversions.asList + /** * This class handles the consumers interaction with zookeeper @@ -68,14 +70,14 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, def createMessageStreams[T]( topicCountMap: java.util.Map[String,java.lang.Integer], decoder: Decoder[T]) - : java.util.Map[String,java.util.List[KafkaMessageStream[T]]] = { + : java.util.Map[String,java.util.List[KafkaStream[T]]] = { import scala.collection.JavaConversions._ val scalaTopicCountMap: Map[String, Int] = Map.empty[String, Int] ++ asMap(topicCountMap.asInstanceOf[java.util.Map[String, Int]]) val scalaReturn = underlying.consume(scalaTopicCountMap, decoder) - val ret = new java.util.HashMap[String,java.util.List[KafkaMessageStream[T]]] + val ret = new java.util.HashMap[String,java.util.List[KafkaStream[T]]] for ((topic, streams) <- scalaReturn) { - var javaStreamList = new java.util.ArrayList[KafkaMessageStream[T]] + var javaStreamList = new java.util.ArrayList[KafkaStream[T]] for (stream <- streams) javaStreamList.add(stream) ret.put(topic, javaStreamList) @@ -85,9 +87,17 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, def createMessageStreams( topicCountMap: java.util.Map[String,java.lang.Integer]) - : java.util.Map[String,java.util.List[KafkaMessageStream[Message]]] = + : java.util.Map[String,java.util.List[KafkaStream[Message]]] = createMessageStreams(topicCountMap, new DefaultDecoder) + def createMessageStreamsByFilter[T](topicFilter: TopicFilter, numStreams: Int, decoder: Decoder[T]) = + asList(underlying.createMessageStreamsByFilter(topicFilter, numStreams, decoder)) + + def createMessageStreamsByFilter(topicFilter: TopicFilter, numStreams: Int) = + createMessageStreamsByFilter(topicFilter, numStreams, new DefaultDecoder) + + def createMessageStreamsByFilter(topicFilter: TopicFilter) = + createMessageStreamsByFilter(topicFilter, 1, new DefaultDecoder) def commitOffsets() { underlying.commitOffsets diff --git core/src/main/scala/kafka/javaapi/message/MessageSet.scala core/src/main/scala/kafka/javaapi/message/MessageSet.scala index 817d9dd..9c9c72f 100644 --- core/src/main/scala/kafka/javaapi/message/MessageSet.scala +++ core/src/main/scala/kafka/javaapi/message/MessageSet.scala @@ -17,8 +17,10 @@ package kafka.javaapi.message + import kafka.message.{MessageAndOffset, InvalidMessageException} + /** * A set of messages. A message set has a fixed serialized form, though the container * for the bytes could be either in-memory or on disk. A The format of each message is diff --git core/src/main/scala/kafka/javaapi/producer/async/CallbackHandler.java core/src/main/scala/kafka/javaapi/producer/async/CallbackHandler.java index 0e0df3d..2b93974 100644 --- core/src/main/scala/kafka/javaapi/producer/async/CallbackHandler.java +++ core/src/main/scala/kafka/javaapi/producer/async/CallbackHandler.java @@ -16,9 +16,9 @@ */ package kafka.javaapi.producer.async; -import kafka.producer.async.QueueItem; import java.util.Properties; +import kafka.producer.async.QueueItem; /** * Callback handler APIs for use in the async producer. The purpose is to diff --git core/src/main/scala/kafka/javaapi/producer/async/EventHandler.java core/src/main/scala/kafka/javaapi/producer/async/EventHandler.java index 381f8e2..842799d 100644 --- core/src/main/scala/kafka/javaapi/producer/async/EventHandler.java +++ core/src/main/scala/kafka/javaapi/producer/async/EventHandler.java @@ -16,12 +16,12 @@ */ package kafka.javaapi.producer.async; -import kafka.javaapi.producer.SyncProducer; -import kafka.producer.async.QueueItem; -import kafka.serializer.Encoder; import java.util.List; import java.util.Properties; +import kafka.javaapi.producer.SyncProducer; +import kafka.producer.async.QueueItem; +import kafka.serializer.Encoder; /** * Handler that dispatches the batched data from the queue of the diff --git core/src/main/scala/kafka/message/ByteBufferMessageSet.scala core/src/main/scala/kafka/message/ByteBufferMessageSet.scala index 07c5b1f..5afd6e1 100644 --- core/src/main/scala/kafka/message/ByteBufferMessageSet.scala +++ core/src/main/scala/kafka/message/ByteBufferMessageSet.scala @@ -36,7 +36,6 @@ import kafka.common.{MessageSizeTooLargeException, InvalidMessageSizeException, class ByteBufferMessageSet(private val buffer: ByteBuffer, private val initialOffset: Long = 0L, private val errorCode: Int = ErrorMapping.NoError) extends MessageSet with Logging { - private var validByteCount = -1L private var shallowValidByteCount = -1L if(sizeInBytes > Int.MaxValue) throw new InvalidMessageSizeException("Message set cannot be larger than " + Int.MaxValue) diff --git core/src/main/scala/kafka/message/MessageAndMetadata.scala core/src/main/scala/kafka/message/MessageAndMetadata.scala new file mode 100644 index 0000000..710308e --- /dev/null +++ core/src/main/scala/kafka/message/MessageAndMetadata.scala @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.message + +case class MessageAndMetadata[T](message: T, topic: String = "") + diff --git core/src/main/scala/kafka/message/MessageAndOffset.scala core/src/main/scala/kafka/message/MessageAndOffset.scala index 1429bb2..d769fc6 100644 --- core/src/main/scala/kafka/message/MessageAndOffset.scala +++ core/src/main/scala/kafka/message/MessageAndOffset.scala @@ -13,11 +13,10 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. -*/ + */ package kafka.message -/** - * Represents message and offset of the next message. This is used in the MessageSet to iterate over it - */ -case class MessageAndOffset(val message: Message, val offset: Long) + +case class MessageAndOffset(message: Message, offset: Long) + diff --git core/src/main/scala/kafka/producer/KafkaLog4jAppender.scala core/src/main/scala/kafka/producer/KafkaLog4jAppender.scala index 5279284..417da27 100644 --- core/src/main/scala/kafka/producer/KafkaLog4jAppender.scala +++ core/src/main/scala/kafka/producer/KafkaLog4jAppender.scala @@ -22,9 +22,7 @@ import org.apache.log4j.spi.LoggingEvent import org.apache.log4j.AppenderSkeleton import org.apache.log4j.helpers.LogLog import kafka.utils.Logging -import kafka.serializer.Encoder import java.util.{Properties, Date} -import kafka.message.Message import scala.collection._ class KafkaLog4jAppender extends AppenderSkeleton with Logging { diff --git core/src/main/scala/kafka/producer/Producer.scala core/src/main/scala/kafka/producer/Producer.scala index 7e0a9f5..dafa6d2 100644 --- core/src/main/scala/kafka/producer/Producer.scala +++ core/src/main/scala/kafka/producer/Producer.scala @@ -22,7 +22,7 @@ import kafka.utils._ import java.util.Properties import kafka.cluster.{Partition, Broker} import java.util.concurrent.atomic.AtomicBoolean -import kafka.common.{NoBrokersForPartitionException, InvalidConfigException, InvalidPartitionException} +import kafka.common.{NoBrokersForPartitionException, InvalidPartitionException} import kafka.api.ProducerRequest class Producer[K,V](config: ProducerConfig, diff --git core/src/main/scala/kafka/server/KafkaServerStartable.scala core/src/main/scala/kafka/server/KafkaServerStartable.scala index 3e196e8..370e20d 100644 --- core/src/main/scala/kafka/server/KafkaServerStartable.scala +++ core/src/main/scala/kafka/server/KafkaServerStartable.scala @@ -17,36 +17,21 @@ package kafka.server -import kafka.utils.{Utils, Logging} -import kafka.consumer._ -import kafka.producer.{ProducerData, ProducerConfig, Producer} -import kafka.message.Message -import java.util.concurrent.CountDownLatch +import kafka.utils.Logging -import scala.collection.Map -class KafkaServerStartable(val serverConfig: KafkaConfig, - val consumerConfig: ConsumerConfig, - val producerConfig: ProducerConfig) extends Logging { +class KafkaServerStartable(val serverConfig: KafkaConfig) extends Logging { private var server : KafkaServer = null - private var embeddedConsumer : EmbeddedConsumer = null init - def this(serverConfig: KafkaConfig) = this(serverConfig, null, null) - private def init() { server = new KafkaServer(serverConfig) - if (consumerConfig != null) - embeddedConsumer = - new EmbeddedConsumer(consumerConfig, producerConfig, this) } def startup() { try { server.startup() - if (embeddedConsumer != null) - embeddedConsumer.startup() } catch { case e => @@ -57,8 +42,6 @@ class KafkaServerStartable(val serverConfig: KafkaConfig, def shutdown() { try { - if (embeddedConsumer != null) - embeddedConsumer.shutdown() server.shutdown() } catch { @@ -73,153 +56,4 @@ class KafkaServerStartable(val serverConfig: KafkaConfig, } } -class EmbeddedConsumer(private val consumerConfig: ConsumerConfig, - private val producerConfig: ProducerConfig, - private val kafkaServerStartable: KafkaServerStartable) extends TopicEventHandler[String] with Logging { - - private val whiteListTopics = - consumerConfig.mirrorTopicsWhitelist.split(",").toList.map(_.trim) - - private val blackListTopics = - consumerConfig.mirrorTopicsBlackList.split(",").toList.map(_.trim) - - // mirrorTopics should be accessed by handleTopicEvent only - private var mirrorTopics:Seq[String] = List() - - private var consumerConnector: ConsumerConnector = null - private var topicEventWatcher:ZookeeperTopicEventWatcher = null - - private val producer = new Producer[Null, Message](producerConfig) - - var threadList = List[MirroringThread]() - - private def isTopicAllowed(topic: String) = { - if (consumerConfig.mirrorTopicsWhitelist.nonEmpty) - whiteListTopics.contains(topic) - else - !blackListTopics.contains(topic) - } - - // TopicEventHandler call-back only - @Override - def handleTopicEvent(allTopics: Seq[String]) { - val newMirrorTopics = allTopics.filter(isTopicAllowed) - - val addedTopics = newMirrorTopics filterNot (mirrorTopics contains) - if (addedTopics.nonEmpty) - info("topic event: added topics = %s".format(addedTopics)) - - val deletedTopics = mirrorTopics filterNot (newMirrorTopics contains) - if (deletedTopics.nonEmpty) - info("topic event: deleted topics = %s".format(deletedTopics)) - - mirrorTopics = newMirrorTopics - - if (addedTopics.nonEmpty || deletedTopics.nonEmpty) { - info("mirror topics = %s".format(mirrorTopics)) - startNewConsumerThreads(makeTopicMap(mirrorTopics)) - } - } - - private def makeTopicMap(mirrorTopics: Seq[String]) = { - if (mirrorTopics.nonEmpty) - Utils.getConsumerTopicMap(mirrorTopics.mkString( - "", ":%d,".format(consumerConfig.mirrorConsumerNumThreads), - ":%d".format(consumerConfig.mirrorConsumerNumThreads))) - else - Utils.getConsumerTopicMap("") - } - - private def startNewConsumerThreads(topicMap: Map[String, Int]) { - if (topicMap.nonEmpty) { - if (consumerConnector != null) - consumerConnector.shutdown() - - /** - * Before starting new consumer threads for the updated set of topics, - * shutdown the existing mirroring threads. Since the consumer connector - * is already shutdown, the mirroring threads should finish their task almost - * instantaneously. If they don't, this points to an error that needs to be looked - * into, and further mirroring should stop - */ - threadList.foreach(_.shutdown) - - // KAFKA: 212: clear the thread list to remove the older thread references that are already shutdown - threadList = Nil - - consumerConnector = Consumer.create(consumerConfig) - val topicMessageStreams = consumerConnector.createMessageStreams(topicMap) - for ((topic, streamList) <- topicMessageStreams) - for (i <- 0 until streamList.length) - threadList ::= new MirroringThread(streamList(i), topic, i) - - threadList.foreach(_.start) - } - else - info("Not starting mirroring threads (mirror topic list is empty)") - } - - def startup() { - info("staring up embedded consumer") - topicEventWatcher = new ZookeeperTopicEventWatcher(consumerConfig, this, kafkaServerStartable) - /* - * consumer threads are (re-)started upon topic events (which includes an - * initial startup event which lists the current topics) - */ - } - - def shutdown() { - // first shutdown the topic watcher to prevent creating new consumer streams - if (topicEventWatcher != null) - topicEventWatcher.shutdown() - info("Stopped the ZK watcher for new topics, now stopping the Kafka consumers") - // stop pulling more data for mirroring - if (consumerConnector != null) - consumerConnector.shutdown() - info("Stopped the kafka consumer threads for existing topics, now stopping the existing mirroring threads") - // wait for all mirroring threads to stop - threadList.foreach(_.shutdown) - info("Stopped all existing mirroring threads, now stopping the producer") - // only then, shutdown the producer - producer.close() - info("Successfully shutdown this Kafka mirror") - } - - class MirroringThread(val stream: KafkaMessageStream[Message], val topic: String, val threadId: Int) extends Thread with Logging { - val shutdownComplete = new CountDownLatch(1) - val name = "kafka-embedded-consumer-%s-%d".format(topic, threadId) - this.setDaemon(false) - this.setName(name) - - - override def run = { - info("Starting mirroring thread %s for topic %s and stream %d".format(name, topic, threadId)) - - try { - for (message <- stream) { - trace("Mirroring thread received message " + message.checksum) - val pd = new ProducerData[Null, Message](topic, message) - producer.send(pd) - } - } - catch { - case e => - fatal(topic + " stream " + threadId + " unexpectedly exited", e) - }finally { - shutdownComplete.countDown - info("Stopped mirroring thread %s for topic %s and stream %d".format(name, topic, threadId)) - } - } - - def shutdown = { - try { - shutdownComplete.await - }catch { - case e: InterruptedException => fatal("Shutdown of thread " + name + " interrupted. " + - "Mirroring thread might leak data!") - } - } - } -} - diff --git core/src/main/scala/kafka/tools/ConsumerShell.scala core/src/main/scala/kafka/tools/ConsumerShell.scala index 53f59b7..5eb5269 100644 --- core/src/main/scala/kafka/tools/ConsumerShell.scala +++ core/src/main/scala/kafka/tools/ConsumerShell.scala @@ -82,15 +82,15 @@ object ConsumerShell { } } -class ZKConsumerThread(stream: KafkaMessageStream[String]) extends Thread with Logging { +class ZKConsumerThread(stream: KafkaStream[String]) extends Thread with Logging { val shutdownLatch = new CountDownLatch(1) override def run() { println("Starting consumer thread..") var count: Int = 0 try { - for (message <- stream) { - println("consumed: " + message) + for (messageAndMetadata <- stream) { + println("consumed: " + messageAndMetadata.message) count += 1 } }catch { diff --git core/src/main/scala/kafka/tools/MirrorMaker.scala core/src/main/scala/kafka/tools/MirrorMaker.scala new file mode 100644 index 0000000..98dd65d --- /dev/null +++ core/src/main/scala/kafka/tools/MirrorMaker.scala @@ -0,0 +1,162 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.tools + +import kafka.message.Message +import joptsimple.OptionParser +import kafka.utils.{Utils, Logging} +import kafka.producer.{ProducerData, ProducerConfig, Producer} +import scala.collection.JavaConversions._ +import java.util.concurrent.CountDownLatch +import kafka.consumer._ + + +object MirrorMaker extends Logging { + + def main(args: Array[String]) { + + info ("Starting mirror maker") + val parser = new OptionParser + + val consumerConfigOpt = parser.accepts("consumer-config", + "Consumer config to consume from a source cluster. " + + "You may specify multiple of these.") + .withRequiredArg() + .describedAs("config file") + .ofType(classOf[String]) + + val producerConfigOpt = parser.accepts("producer-config", + "Embedded producer config.") + .withRequiredArg() + .describedAs("config file") + .ofType(classOf[String]) + + val numStreamsOpt = parser.accepts("num-streams", + "Number of consumption streams.") + .withRequiredArg() + .describedAs("Number of threads") + .ofType(classOf[java.lang.Integer]) + .defaultsTo(1) + + val whitelistOpt = parser.accepts("whitelist", + "Whitelist of topics to mirror.") + .withRequiredArg() + .describedAs("Java regex (String)") + .ofType(classOf[String]) + + val blacklistOpt = parser.accepts("blacklist", + "Blacklist of topics to mirror.") + .withRequiredArg() + .describedAs("Java regex (String)") + .ofType(classOf[String]) + + val helpOpt = parser.accepts("help", "Print this message.") + + val options = parser.parse(args : _*) + + if (options.has(helpOpt)) { + parser.printHelpOn(System.out) + System.exit(0) + } + + Utils.checkRequiredArgs( + parser, options, consumerConfigOpt, producerConfigOpt) + if (List(whitelistOpt, blacklistOpt).count(options.has) != 1) { + println("Exactly one of whitelist or blacklist is required.") + System.exit(1) + } + + val numStreams = options.valueOf(numStreamsOpt) + + val producer = { + val config = new ProducerConfig( + Utils.loadProps(options.valueOf(producerConfigOpt))) + new Producer[Null, Message](config) + } + + val threads = { + val connectors = options.valuesOf(consumerConfigOpt).toList + .map(cfg => new ConsumerConfig(Utils.loadProps(cfg.toString))) + .map(new ZookeeperConsumerConnector(_)) + + Runtime.getRuntime.addShutdownHook(new Thread() { + override def run() { + connectors.foreach(_.shutdown()) + producer.close() + } + }) + + val filterSpec = if (options.has(whitelistOpt)) + new Whitelist(options.valueOf(whitelistOpt)) + else + new Blacklist(options.valueOf(blacklistOpt)) + + val streams = + connectors.map(_.createMessageStreamsByFilter(filterSpec, numStreams.intValue())) + + streams.flatten.zipWithIndex.map(streamAndIndex => { + new MirrorMakerThread(streamAndIndex._1, producer, streamAndIndex._2) + }) + } + + threads.foreach(_.start()) + + threads.foreach(_.awaitShutdown()) + } + + class MirrorMakerThread(stream: KafkaStream[Message], + producer: Producer[Null, Message], + threadId: Int) + extends Thread with Logging { + + private val shutdownLatch = new CountDownLatch(1) + private val threadName = "mirrormaker-" + threadId + + this.setName(threadName) + + override def run() { + try { + for (msgAndMetadata <- stream) { + val pd = new ProducerData[Null, Message]( + msgAndMetadata.topic, msgAndMetadata.message) + producer.send(pd) + } + } + catch { + case e => + fatal("%s stream unexpectedly exited.", e) + } + finally { + shutdownLatch.countDown() + info("Stopped thread %s.".format(threadName)) + } + } + + def awaitShutdown() { + try { + shutdownLatch.await() + } + catch { + case e: InterruptedException => fatal( + "Shutdown of thread %s interrupted. This might leak data!" + .format(threadName)) + } + } + } +} + diff --git core/src/main/scala/kafka/tools/ReplayLogProducer.scala core/src/main/scala/kafka/tools/ReplayLogProducer.scala index 79d3998..1300cf6 100644 --- core/src/main/scala/kafka/tools/ReplayLogProducer.scala +++ core/src/main/scala/kafka/tools/ReplayLogProducer.scala @@ -34,8 +34,6 @@ object ReplayLogProducer extends Logging { private val GROUPID: String = "replay-log-producer" def main(args: Array[String]) { - var isNoPrint = false; - val config = new Config(args) val executor = Executors.newFixedThreadPool(config.numThreads) @@ -153,7 +151,7 @@ object ReplayLogProducer extends Logging { } } - class ZKConsumerThread(config: Config, stream: KafkaMessageStream[Message]) extends Thread with Logging { + class ZKConsumerThread(config: Config, stream: KafkaStream[Message]) extends Thread with Logging { val shutdownLatch = new CountDownLatch(1) val props = new Properties() val brokerInfoList = config.brokerInfo.split("=") @@ -184,9 +182,9 @@ object ReplayLogProducer extends Logging { stream.slice(0, config.numMessages) else stream - for (message <- iter) { + for (messageAndMetadata <- iter) { try { - producer.send(new ProducerData[Message, Message](config.outputTopic, message)) + producer.send(new ProducerData[Message, Message](config.outputTopic, messageAndMetadata.message)) if (config.delayedMSBtwSend > 0 && (messageCount + 1) % config.batchSize == 0) Thread.sleep(config.delayedMSBtwSend) messageCount += 1 diff --git core/src/main/scala/kafka/utils/Logging.scala core/src/main/scala/kafka/utils/Logging.scala index 2e664f5..6e05eb4 100644 --- core/src/main/scala/kafka/utils/Logging.scala +++ core/src/main/scala/kafka/utils/Logging.scala @@ -23,72 +23,76 @@ trait Logging { val loggerName = this.getClass.getName lazy val logger = Logger.getLogger(loggerName) + protected var logIdent = "" + + private def msgWithLogIdent(msg: String) = "%s%s".format(logIdent, msg) + def trace(msg: => String): Unit = { if (logger.isTraceEnabled()) - logger.trace(msg) + logger.trace(msgWithLogIdent(msg)) } def trace(e: => Throwable): Any = { if (logger.isTraceEnabled()) - logger.trace("",e) + logger.trace(logIdent,e) } def trace(msg: => String, e: => Throwable) = { if (logger.isTraceEnabled()) - logger.trace(msg,e) + logger.trace(msgWithLogIdent(msg),e) } def debug(msg: => String): Unit = { if (logger.isDebugEnabled()) - logger.debug(msg) + logger.debug(msgWithLogIdent(msg)) } def debug(e: => Throwable): Any = { if (logger.isDebugEnabled()) - logger.debug("",e) + logger.debug(logIdent,e) } def debug(msg: => String, e: => Throwable) = { if (logger.isDebugEnabled()) - logger.debug(msg,e) + logger.debug(msgWithLogIdent(msg),e) } def info(msg: => String): Unit = { if (logger.isInfoEnabled()) - logger.info(msg) + logger.info(msgWithLogIdent(msg)) } def info(e: => Throwable): Any = { if (logger.isInfoEnabled()) - logger.info("",e) + logger.info(logIdent,e) } def info(msg: => String,e: => Throwable) = { if (logger.isInfoEnabled()) - logger.info(msg,e) + logger.info(msgWithLogIdent(msg),e) } def warn(msg: => String): Unit = { - logger.warn(msg) + logger.warn(msgWithLogIdent(msg)) } def warn(e: => Throwable): Any = { - logger.warn("",e) + logger.warn(logIdent,e) } def warn(msg: => String, e: => Throwable) = { - logger.warn(msg,e) + logger.warn(msgWithLogIdent(msg),e) } def error(msg: => String): Unit = { - logger.error(msg) + logger.error(msgWithLogIdent(msg)) } def error(e: => Throwable): Any = { - logger.error("",e) + logger.error(logIdent,e) } def error(msg: => String, e: => Throwable) = { - logger.error(msg,e) + logger.error(msgWithLogIdent(msg),e) } def fatal(msg: => String): Unit = { - logger.fatal(msg) + logger.fatal(msgWithLogIdent(msg)) } def fatal(e: => Throwable): Any = { - logger.fatal("",e) + logger.fatal(logIdent,e) } def fatal(msg: => String, e: => Throwable) = { - logger.fatal(msg,e) + logger.fatal(msgWithLogIdent(msg),e) } } diff --git core/src/main/scala/kafka/utils/Utils.scala core/src/main/scala/kafka/utils/Utils.scala index 7b8b5ae..a3c2701 100644 --- core/src/main/scala/kafka/utils/Utils.scala +++ core/src/main/scala/kafka/utils/Utils.scala @@ -29,12 +29,13 @@ import scala.collection._ import scala.collection.mutable import kafka.message.{NoCompressionCodec, CompressionCodec} import org.I0Itec.zkclient.ZkClient +import joptsimple.{OptionSpec, OptionSet, OptionParser} + /** * Helper functions! */ object Utils extends Logging { - /** * Wrap the given function in a java.lang.Runnable * @param fun A function @@ -657,6 +658,16 @@ object Utils extends Logging { case _ => // swallow } } + + def checkRequiredArgs(parser: OptionParser, options: OptionSet, required: OptionSpec[_]*) { + for(arg <- required) { + if(!options.has(arg)) { + error("Missing required argument \"" + arg + "\"") + parser.printHelpOn(System.err) + System.exit(1) + } + } + } } class SnapshotStats(private val monitorDurationNs: Long = 600L * 1000L * 1000L * 1000L) { diff --git core/src/main/scala/kafka/utils/ZkUtils.scala core/src/main/scala/kafka/utils/ZkUtils.scala index 917da0f..caddb06 100644 --- core/src/main/scala/kafka/utils/ZkUtils.scala +++ core/src/main/scala/kafka/utils/ZkUtils.scala @@ -243,17 +243,11 @@ object ZkUtils extends Logging { getChildren(zkClient, dirs.consumerRegistryDir) } - def getTopicCount(zkClient: ZkClient, group: String, consumerId: String) : TopicCount = { - val dirs = new ZKGroupDirs(group) - val topicCountJson = ZkUtils.readData(zkClient, dirs.consumerRegistryDir + "/" + consumerId) - TopicCount.constructTopicCount(consumerId, topicCountJson) - } - def getConsumerTopicMaps(zkClient: ZkClient, group: String): Map[String, TopicCount] = { val dirs = new ZKGroupDirs(group) val consumersInGroup = getConsumersInGroup(zkClient, group) val topicCountMaps = consumersInGroup.map(consumerId => TopicCount.constructTopicCount(consumerId, - ZkUtils.readData(zkClient, dirs.consumerRegistryDir + "/" + consumerId))) + ZkUtils.readData(zkClient, dirs.consumerRegistryDir + "/" + consumerId), zkClient)) consumersInGroup.zip(topicCountMaps).toMap } @@ -262,8 +256,8 @@ object ZkUtils extends Logging { val consumers = getChildrenParentMayNotExist(zkClient, dirs.consumerRegistryDir) val consumersPerTopicMap = new mutable.HashMap[String, List[String]] for (consumer <- consumers) { - val topicCount = getTopicCount(zkClient, group, consumer) - for ((topic, consumerThreadIdSet) <- topicCount.getConsumerThreadIdsPerTopic()) { + val topicCount = TopicCount.constructTopicCount(group, consumer, zkClient) + for ((topic, consumerThreadIdSet) <- topicCount.getConsumerThreadIdsPerTopic) { for (consumerThreadId <- consumerThreadIdSet) consumersPerTopicMap.get(topic) match { case Some(curConsumers) => consumersPerTopicMap.put(topic, consumerThreadId :: curConsumers) diff --git core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala index c8e4a3c..fa709de 100644 --- core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala +++ core/src/test/scala/other/kafka/TestZKConsumerOffsets.scala @@ -56,13 +56,13 @@ object TestZKConsumerOffsets { } } -private class ConsumerThread(stream: KafkaMessageStream[Message]) extends Thread { +private class ConsumerThread(stream: KafkaStream[Message]) extends Thread { val shutdownLatch = new CountDownLatch(1) override def run() { println("Starting consumer thread..") - for (message <- stream) { - println("consumed: " + Utils.toString(message.payload, "UTF-8")) + for (messageAndMetadata <- stream) { + println("consumed: " + Utils.toString(messageAndMetadata.message.payload, "UTF-8")) } shutdownLatch.countDown println("thread shutdown !" ) diff --git core/src/test/scala/unit/kafka/consumer/TopicCountTest.scala core/src/test/scala/unit/kafka/consumer/TopicCountTest.scala deleted file mode 100644 index 1813444..0000000 --- core/src/test/scala/unit/kafka/consumer/TopicCountTest.scala +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.consumer - -import junit.framework.Assert._ -import org.junit.Test -import org.scalatest.junit.JUnitSuite -import kafka.cluster.Partition - - -class TopicCountTest extends JUnitSuite { - - @Test - def testBasic() { - val consumer = "conusmer1" - val json = """{ "topic1" : 2, "topic2" : 3 }""" - val topicCount = TopicCount.constructTopicCount(consumer, json) - val topicCountMap = Map( - "topic1" -> 2, - "topic2" -> 3 - ) - val expectedTopicCount = new TopicCount(consumer, topicCountMap) - assertTrue(expectedTopicCount == topicCount) - - val topicCount2 = TopicCount.constructTopicCount(consumer, expectedTopicCount.toJsonString) - assertTrue(expectedTopicCount == topicCount2) - } - - @Test - def testPartition() { - assertTrue(new Partition(10, 0) == new Partition(10, 0)) - assertTrue(new Partition(10, 1) != new Partition(10, 0)) - } -} diff --git core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala new file mode 100644 index 0000000..40a2bf7 --- /dev/null +++ core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.consumer + + +import junit.framework.Assert._ +import org.scalatest.junit.JUnitSuite +import org.junit.Test + + +class TopicFilterTest extends JUnitSuite { + + @Test + def testWhitelists() { + + val topicFilter1 = new Whitelist("white1,white2") + assertFalse(topicFilter1.requiresTopicEventWatcher) + assertTrue(topicFilter1.isTopicAllowed("white2")) + assertFalse(topicFilter1.isTopicAllowed("black1")) + + val topicFilter2 = new Whitelist(".+") + assertTrue(topicFilter2.requiresTopicEventWatcher) + assertTrue(topicFilter2.isTopicAllowed("alltopics")) + + val topicFilter3 = new Whitelist("white_listed-topic.+") + assertTrue(topicFilter3.requiresTopicEventWatcher) + assertTrue(topicFilter3.isTopicAllowed("white_listed-topic1")) + assertFalse(topicFilter3.isTopicAllowed("black1")) + } + + @Test + def testBlacklists() { + val topicFilter1 = new Blacklist("black1") + assertTrue(topicFilter1.requiresTopicEventWatcher) + } +} \ No newline at end of file diff --git core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala index e255b7d..0df05d3 100644 --- core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala +++ core/src/test/scala/unit/kafka/consumer/ZookeeperConsumerConnectorTest.scala @@ -234,7 +234,7 @@ class ZookeeperConsumerConnectorTest extends JUnit3Suite with KafkaServerTestHar val iterator = messageStream.iterator for (i <- 0 until nMessages * 2) { assertTrue(iterator.hasNext()) - val message = iterator.next() + val message = iterator.next().message receivedMessages ::= message debug("received message: " + message) } @@ -270,14 +270,14 @@ class ZookeeperConsumerConnectorTest extends JUnit3Suite with KafkaServerTestHar messages.sortWith((s,t) => s.checksum < t.checksum) } - def getMessages(nMessagesPerThread: Int, topicMessageStreams: Map[String,List[KafkaMessageStream[Message]]]): List[Message]= { + def getMessages(nMessagesPerThread: Int, topicMessageStreams: Map[String,List[KafkaStream[Message]]]): List[Message]= { var messages: List[Message] = Nil for ((topic, messageStreams) <- topicMessageStreams) { for (messageStream <- messageStreams) { val iterator = messageStream.iterator for (i <- 0 until nMessagesPerThread) { assertTrue(iterator.hasNext) - val message = iterator.next + val message = iterator.next.message messages ::= message debug("received message: " + Utils.toString(message.payload, "UTF-8")) } diff --git core/src/test/scala/unit/kafka/integration/FetcherTest.scala core/src/test/scala/unit/kafka/integration/FetcherTest.scala index 40b7ff4..915af85 100644 --- core/src/test/scala/unit/kafka/integration/FetcherTest.scala +++ core/src/test/scala/unit/kafka/integration/FetcherTest.scala @@ -56,7 +56,7 @@ class FetcherTest extends JUnit3Suite with KafkaServerTestHarness { super.setUp fetcher = new Fetcher(new ConsumerConfig(TestUtils.createConsumerProperties("", "", "")), null) fetcher.stopConnectionsToAllBrokers - fetcher.startConnections(topicInfos, cluster, null) + fetcher.startConnections(topicInfos, cluster) } override def tearDown() { diff --git core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala index c7653e5..f7a4b15 100644 --- core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala +++ core/src/test/scala/unit/kafka/javaapi/consumer/ZookeeperConsumerConnectorTest.scala @@ -26,9 +26,10 @@ import kafka.utils.{TestZKUtils, TestUtils} import org.scalatest.junit.JUnit3Suite import scala.collection.JavaConversions._ import kafka.javaapi.message.ByteBufferMessageSet -import kafka.consumer.{ConsumerConfig, KafkaMessageStream} import org.apache.log4j.{Level, Logger} import kafka.message.{NoCompressionCodec, CompressionCodec, Message} +import kafka.consumer.{KafkaStream, ConsumerConfig} + class ZookeeperConsumerConnectorTest extends JUnit3Suite with KafkaServerTestHarness with ZooKeeperTestHarness with Logging { @@ -91,7 +92,7 @@ class ZookeeperConsumerConnectorTest extends JUnit3Suite with KafkaServerTestHar messages.sortWith((s,t) => s.checksum < t.checksum) } - def getMessages(nMessagesPerThread: Int, jTopicMessageStreams: java.util.Map[String, java.util.List[KafkaMessageStream[Message]]]) + def getMessages(nMessagesPerThread: Int, jTopicMessageStreams: java.util.Map[String, java.util.List[KafkaStream[Message]]]) : List[Message]= { var messages: List[Message] = Nil val topicMessageStreams = asMap(jTopicMessageStreams) @@ -100,7 +101,7 @@ class ZookeeperConsumerConnectorTest extends JUnit3Suite with KafkaServerTestHar val iterator = messageStream.iterator for (i <- 0 until nMessagesPerThread) { assertTrue(iterator.hasNext) - val message = iterator.next + val message = iterator.next.message messages ::= message debug("received message: " + Utils.toString(message.payload, "UTF-8")) } diff --git core/src/test/scala/unit/kafka/utils/UtilsTest.scala core/src/test/scala/unit/kafka/utils/UtilsTest.scala index 7e75f8f..218e229 100644 --- core/src/test/scala/unit/kafka/utils/UtilsTest.scala +++ core/src/test/scala/unit/kafka/utils/UtilsTest.scala @@ -21,6 +21,7 @@ import org.apache.log4j.Logger import org.scalatest.junit.JUnitSuite import org.junit.Test + class UtilsTest extends JUnitSuite { private val logger = Logger.getLogger(classOf[UtilsTest]) @@ -29,5 +30,5 @@ class UtilsTest extends JUnitSuite { def testSwallow() { Utils.swallow(logger.info, throw new IllegalStateException("test")) } - + } diff --git examples/src/main/java/kafka/examples/Consumer.java examples/src/main/java/kafka/examples/Consumer.java index 18b7348..cb01577 100644 --- examples/src/main/java/kafka/examples/Consumer.java +++ examples/src/main/java/kafka/examples/Consumer.java @@ -16,16 +16,17 @@ */ package kafka.examples; -import kafka.consumer.ConsumerConfig; -import kafka.consumer.ConsumerIterator; -import kafka.consumer.KafkaMessageStream; -import kafka.javaapi.consumer.ConsumerConnector; -import kafka.message.Message; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import kafka.consumer.ConsumerConfig; +import kafka.consumer.ConsumerIterator; +import kafka.consumer.KafkaStream; +import kafka.javaapi.consumer.ConsumerConnector; +import kafka.message.Message; + public class Consumer extends Thread { @@ -55,10 +56,10 @@ public class Consumer extends Thread public void run() { Map topicCountMap = new HashMap(); topicCountMap.put(topic, new Integer(1)); - Map>> consumerMap = consumer.createMessageStreams(topicCountMap); - KafkaMessageStream stream = consumerMap.get(topic).get(0); + Map>> consumerMap = consumer.createMessageStreams(topicCountMap); + KafkaStream stream = consumerMap.get(topic).get(0); ConsumerIterator it = stream.iterator(); while(it.hasNext()) - System.out.println(ExampleUtils.getMessage(it.next())); + System.out.println(ExampleUtils.getMessage(it.next().message())); } } diff --git examples/src/main/java/kafka/examples/ExampleUtils.java examples/src/main/java/kafka/examples/ExampleUtils.java index c301a52..34fd1c0 100644 --- examples/src/main/java/kafka/examples/ExampleUtils.java +++ examples/src/main/java/kafka/examples/ExampleUtils.java @@ -16,8 +16,8 @@ */ package kafka.examples; -import java.nio.ByteBuffer; +import java.nio.ByteBuffer; import kafka.message.Message; public class ExampleUtils diff --git examples/src/main/java/kafka/examples/Producer.java examples/src/main/java/kafka/examples/Producer.java index c663a54..353a7eb 100644 --- examples/src/main/java/kafka/examples/Producer.java +++ examples/src/main/java/kafka/examples/Producer.java @@ -16,9 +16,10 @@ */ package kafka.examples; + +import java.util.Properties; import kafka.javaapi.producer.ProducerData; import kafka.producer.ProducerConfig; -import java.util.Properties; public class Producer extends Thread { diff --git examples/src/main/java/kafka/examples/SimpleConsumerDemo.java examples/src/main/java/kafka/examples/SimpleConsumerDemo.java index 1cb8a83..c2b88da 100644 --- examples/src/main/java/kafka/examples/SimpleConsumerDemo.java +++ examples/src/main/java/kafka/examples/SimpleConsumerDemo.java @@ -16,17 +16,14 @@ */ package kafka.examples; + import java.util.ArrayList; import java.util.List; - +import kafka.api.FetchRequest; import kafka.javaapi.MultiFetchResponse; import kafka.javaapi.consumer.SimpleConsumer; import kafka.javaapi.message.ByteBufferMessageSet; import kafka.message.MessageAndOffset; -import scala.collection.Iterator; - -import kafka.api.FetchRequest; -import kafka.message.Message; public class SimpleConsumerDemo diff --git perf/src/main/scala/kafka/perf/ConsumerPerformance.scala perf/src/main/scala/kafka/perf/ConsumerPerformance.scala index 541bf42..414c965 100644 --- perf/src/main/scala/kafka/perf/ConsumerPerformance.scala +++ perf/src/main/scala/kafka/perf/ConsumerPerformance.scala @@ -17,15 +17,12 @@ package kafka.perf -import java.net.URI import java.util.concurrent.CountDownLatch import java.util.concurrent.atomic.AtomicLong import java.nio.channels.ClosedByInterruptException -import joptsimple._ import org.apache.log4j.Logger import kafka.message.Message -import org.I0Itec.zkclient.ZkClient -import kafka.utils.{ZKStringSerializer, Utils} +import kafka.utils.Utils import java.util.{Random, Properties} import kafka.consumer._ import java.text.SimpleDateFormat @@ -139,7 +136,7 @@ object ConsumerPerformance { val hideHeader = options.has(hideHeaderOpt) } - class ConsumerPerfThread(threadId: Int, name: String, stream: KafkaMessageStream[Message], + class ConsumerPerfThread(threadId: Int, name: String, stream: KafkaStream[Message], config:ConsumerPerfConfig, totalMessagesRead: AtomicLong, totalBytesRead: AtomicLong) extends Thread(name) { private val shutdownLatch = new CountDownLatch(1) @@ -157,9 +154,9 @@ object ConsumerPerformance { var lastMessagesRead = 0L try { - for (message <- stream if messagesRead < config.numMessages) { + for (messageAndMetadata <- stream if messagesRead < config.numMessages) { messagesRead += 1 - bytesRead += message.payloadSize + bytesRead += messageAndMetadata.message.payloadSize if (messagesRead % config.reportingInterval == 0) { if(config.showDetailedStats) diff --git perf/src/main/scala/kafka/perf/PerfConfig.scala perf/src/main/scala/kafka/perf/PerfConfig.scala index 265caa8..db2c1a1 100644 --- perf/src/main/scala/kafka/perf/PerfConfig.scala +++ perf/src/main/scala/kafka/perf/PerfConfig.scala @@ -18,7 +18,7 @@ package kafka.perf import joptsimple.OptionParser -import java.text.SimpleDateFormat + class PerfConfig(args: Array[String]) { val parser = new OptionParser diff --git perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala index 02c3008..ca8df59 100644 --- perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala +++ perf/src/main/scala/kafka/perf/SimpleConsumerPerformance.scala @@ -18,9 +18,7 @@ package kafka.perf import java.net.URI -import joptsimple._ import kafka.utils._ -import kafka.server._ import kafka.consumer.SimpleConsumer import org.apache.log4j.Logger import kafka.api.{OffsetRequest, FetchRequest} diff --git system_test/embedded_consumer/README system_test/embedded_consumer/README deleted file mode 100644 index 8cd6b88..0000000 --- system_test/embedded_consumer/README +++ /dev/null @@ -1,27 +0,0 @@ -This test replicates messages from 3 kafka brokers to 2 other kafka brokers -using the embedded consumer. At the end, the messages produced at the source -brokers should match that at the target brokers. - -To run this test, do -bin/run-test.sh - -The expected output is given in bin/expected.out. There is only 1 thing that's -important. -1. The output should have a line "test passed". - -In the event of failure, by default the brokers and zookeepers remain running -to make it easier to debug the issue - hit Ctrl-C to shut them down. You can -change this behavior by setting the action_on_fail flag in the script to "exit" -or "proceed", in which case a snapshot of all the logs and directories is -placed in the test's base directory. - -If you are making any changes that may affect the embedded consumer, it is a -good idea to run the test in a loop. E.g.: - -:>/tmp/embeddedconsumer_test.log -for i in {1..10}; do echo "run $i"; ./bin/run-test.sh 2>1 >> /tmp/embeddedconsumer_test.log; done -tail -F /tmp/embeddedconsumer_test.log - -grep -ic passed /tmp/embeddedconsumer_test.log -grep -ic failed /tmp/embeddedconsumer_test.log - diff --git system_test/embedded_consumer/bin/expected.out system_test/embedded_consumer/bin/expected.out deleted file mode 100644 index 0a1bbaf..0000000 --- system_test/embedded_consumer/bin/expected.out +++ /dev/null @@ -1,18 +0,0 @@ -start the servers ... -start producing messages ... -wait for consumer to finish consuming ... -[2011-05-17 14:49:11,605] INFO Creating async producer for broker id = 2 at localhost:9091 (kafka.producer.ProducerPool) -[2011-05-17 14:49:11,606] INFO Creating async producer for broker id = 1 at localhost:9092 (kafka.producer.ProducerPool) -[2011-05-17 14:49:11,607] INFO Creating async producer for broker id = 3 at localhost:9090 (kafka.producer.ProducerPool) -thread 0: 400000 messages sent 3514012.1233 nMsg/sec 3.3453 MBs/sec -[2011-05-17 14:49:34,382] INFO Closing all async producers (kafka.producer.ProducerPool) -[2011-05-17 14:49:34,383] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) -[2011-05-17 14:49:34,384] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) -[2011-05-17 14:49:34,385] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) -Total Num Messages: 400000 bytes: 79859641 in 22.93 secs -Messages/sec: 17444.3960 -MB/sec: 3.3214 -test passed -stopping the servers -bin/../../../bin/zookeeper-server-start.sh: line 9: 22584 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ -bin/../../../bin/zookeeper-server-start.sh: line 9: 22585 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ diff --git system_test/embedded_consumer/bin/run-test.sh system_test/embedded_consumer/bin/run-test.sh deleted file mode 100755 index e11fe27..0000000 --- system_test/embedded_consumer/bin/run-test.sh +++ /dev/null @@ -1,328 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -readonly num_messages=400000 -readonly message_size=400 -readonly action_on_fail="proceed" - -readonly test_start_time="$(date +%s)" - -readonly base_dir=$(dirname $0)/.. - -info() { - echo -e "$(date +"%Y-%m-%d %H:%M:%S") $*" -} - -kill_child_processes() { - isTopmost=$1 - curPid=$2 - childPids=$(ps a -o pid= -o ppid= | grep "${curPid}$" | awk '{print $1;}') - for childPid in $childPids - do - kill_child_processes 0 $childPid - done - if [ $isTopmost -eq 0 ]; then - kill -15 $curPid 2> /dev/null - fi -} - -cleanup() { - info "cleaning up" - - pid_zk_source= - pid_zk_target= - pid_kafka_source1= - pid_kafka_source2= - pid_kafka_source3= - pid_kafka_target1= - pid_kafka_target2= - pid_producer= - - rm -rf /tmp/zookeeper_source - rm -rf /tmp/zookeeper_target - - rm -rf /tmp/kafka-source{1..3}-logs - # mkdir -p /tmp/kafka-source{1..3}-logs/test0{1..3}-0 - # touch /tmp/kafka-source{1..3}-logs/test0{1..3}-0/00000000000000000000.kafka - - rm -rf /tmp/kafka-target{1..2}-logs -} - -begin_timer() { - t_begin=$(date +%s) -} - -end_timer() { - t_end=$(date +%s) -} - -start_zk() { - info "starting zookeepers" - $base_dir/../../bin/zookeeper-server-start.sh $base_dir/config/zookeeper_source.properties 2>&1 > $base_dir/zookeeper_source.log & - pid_zk_source=$! - $base_dir/../../bin/zookeeper-server-start.sh $base_dir/config/zookeeper_target.properties 2>&1 > $base_dir/zookeeper_target.log & - pid_zk_target=$! -} - -start_source_servers() { - info "starting source cluster" - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source1.properties 2>&1 > $base_dir/kafka_source1.log & - pid_kafka_source1=$! - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source2.properties 2>&1 > $base_dir/kafka_source2.log & - pid_kafka_source2=$! - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source3.properties 2>&1 > $base_dir/kafka_source3.log & - pid_kafka_source3=$! -} - -start_target_servers_for_whitelist_test() { - echo "starting mirror cluster" - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target1.properties $base_dir/config/whitelisttest.consumer.properties $base_dir/config/mirror_producer.properties 2>&1 > $base_dir/kafka_target1.log & - pid_kafka_target1=$! - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target2.properties $base_dir/config/whitelisttest.consumer.properties $base_dir/config/mirror_producer.properties 2>&1 > $base_dir/kafka_target2.log & - pid_kafka_target2=$! -} - -start_target_servers_for_blacklist_test() { - echo "starting mirror cluster" - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target1.properties $base_dir/config/blacklisttest.consumer.properties $base_dir/config/mirror_producer.properties 2>&1 > $base_dir/kafka_target1.log & - pid_kafka_target1=$! - $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target2.properties $base_dir/config/blacklisttest.consumer.properties $base_dir/config/mirror_producer.properties 2>&1 > $base_dir/kafka_target2.log & - pid_kafka_target2=$! -} - -shutdown_servers() { - info "stopping producer" - if [ "x${pid_producer}" != "x" ]; then kill_child_processes 0 ${pid_producer}; fi - - info "shutting down target servers" - if [ "x${pid_kafka_target1}" != "x" ]; then kill_child_processes 0 ${pid_kafka_target1}; fi - if [ "x${pid_kafka_target2}" != "x" ]; then kill_child_processes 0 ${pid_kafka_target2}; fi - sleep 2 - - info "shutting down source servers" - if [ "x${pid_kafka_source1}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source1}; fi - if [ "x${pid_kafka_source2}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source2}; fi - if [ "x${pid_kafka_source3}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source3}; fi - - info "shutting down zookeeper servers" - if [ "x${pid_zk_target}" != "x" ]; then kill_child_processes 0 ${pid_zk_target}; fi - if [ "x${pid_zk_source}" != "x" ]; then kill_child_processes 0 ${pid_zk_source}; fi -} - -start_producer() { - topic=$1 - info "start producing messages for topic $topic ..." - $base_dir/../../bin/kafka-run-class.sh kafka.tools.ProducerPerformance --brokerinfo zk.connect=localhost:2181 --topic $topic --messages $num_messages --message-size $message_size --batch-size 200 --vary-message-size --threads 1 --reporting-interval $num_messages --async 2>&1 > $base_dir/producer_performance.log & - pid_producer=$! -} - -# In case the consumer does not consume, the test may exit prematurely (i.e., -# shut down the kafka brokers, and ProducerPerformance will start throwing ugly -# exceptions. So, wait for the producer to finish before shutting down. If it -# takes too long, the user can just hit Ctrl-c which is trapped to kill child -# processes. -# Usage: wait_partition_done ([kafka-server] [topic] [partition-id])+ -wait_partition_done() { - n_tuples=$(($# / 3)) - - i=1 - while (($#)); do - kafka_server[i]=$1 - topic[i]=$2 - partitionid[i]=$3 - prev_offset[i]=0 - info "\twaiting for partition on server ${kafka_server[i]}, topic ${topic[i]}, partition ${partitionid[i]}" - i=$((i+1)) - shift 3 - done - - all_done=0 - - # set -x - while [[ $all_done != 1 ]]; do - sleep 4 - i=$n_tuples - all_done=1 - for ((i=1; i <= $n_tuples; i++)); do - cur_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server ${kafka_server[i]} --topic ${topic[i]} --partition ${partitionid[i]} --time -1 --offsets 1 | tail -1) - if [ "x$cur_size" != "x${prev_offset[i]}" ]; then - all_done=0 - prev_offset[i]=$cur_size - fi - done - done - -} - -cmp_logs() { - topic=$1 - info "comparing source and target logs for topic $topic" - source_part0_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9092 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) - source_part1_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9091 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) - source_part2_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9090 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) - target_part0_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9093 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) - target_part1_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9094 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) - if [ "x$target_part0_size" == "x" ]; then target_part0_size=0; fi - if [ "x$target_part1_size" == "x" ]; then target_part1_size=0; fi - expected_size=$(($source_part0_size + $source_part1_size + $source_part2_size)) - actual_size=$(($target_part0_size + $target_part1_size)) - if [ "x$expected_size" != "x$actual_size" ] - then - info "source size: $expected_size target size: $actual_size" - return 1 - else - return 0 - fi -} - -take_fail_snapshot() { - snapshot_dir="$base_dir/failed-${snapshot_prefix}-${test_start_time}" - mkdir $snapshot_dir - for dir in /tmp/zookeeper_source /tmp/zookeeper_target /tmp/kafka-source{1..3}-logs /tmp/kafka-target{1..2}-logs; do - if [ -d $dir ]; then - cp -r $dir $snapshot_dir - fi - done -} - -# Usage: process_test_result -# result: last test result -# action_on_fail: (exit|wait|proceed) -# ("wait" is useful if you want to troubleshoot using zookeeper) -process_test_result() { - result=$1 - if [ $1 -eq 0 ]; then - info "test passed" - else - info "test failed" - case "$2" in - "wait") info "waiting: hit Ctrl-c to quit" - wait - ;; - "exit") shutdown_servers - take_fail_snapshot - exit $result - ;; - *) shutdown_servers - take_fail_snapshot - info "proceeding" - ;; - esac - fi -} - -test_whitelists() { - info "### Testing whitelists" - snapshot_prefix="whitelist-test" - - cleanup - start_zk - start_source_servers - start_target_servers_for_whitelist_test - sleep 4 - - begin_timer - - start_producer test01 - info "waiting for producer to finish producing ..." - wait_partition_done kafka://localhost:9090 test01 0 kafka://localhost:9091 test01 0 kafka://localhost:9092 test01 0 - - info "waiting for consumer to finish consuming ..." - wait_partition_done kafka://localhost:9093 test01 0 kafka://localhost:9094 test01 0 - - end_timer - info "embedded consumer took $((t_end - t_begin)) seconds" - - sleep 2 - - cmp_logs test01 - result=$? - - return $result -} - -test_blacklists() { - info "### Testing blacklists" - snapshot_prefix="blacklist-test" - cleanup - start_zk - start_source_servers - start_target_servers_for_blacklist_test - sleep 4 - - start_producer test02 - info "waiting for producer to finish producing test02 ..." - wait_partition_done kafka://localhost:9090 test02 0 kafka://localhost:9091 test02 0 kafka://localhost:9092 test02 0 - - # start_producer test03 - # info "waiting for producer to finish producing test03 ..." - # wait_partition_done kafka://localhost:9090 test03 0 kafka://localhost:9091 test03 0 kafka://localhost:9092 test03 0 - - begin_timer - - start_producer test01 - info "waiting for producer to finish producing ..." - wait_partition_done kafka://localhost:9090 test01 0 kafka://localhost:9091 test01 0 kafka://localhost:9092 test01 0 - - info "waiting for consumer to finish consuming ..." - wait_partition_done kafka://localhost:9093 test01 0 kafka://localhost:9094 test01 0 - - end_timer - - info "embedded consumer took $((t_end - t_begin)) seconds" - - sleep 2 - - cmp_logs test02 - result1=$? - # cmp_logs test03 - # result2=$? - # if [[ "x$result1" == "x0" || "x$result2" == "x0" ]]; then - if [[ "x$result1" == "x0" ]]; then - result=1 - else - cmp_logs test01 - result=$? - fi - - return $result -} - -# main test begins - -echo "Test-$test_start_time" - -# Ctrl-c trap. Catches INT signal -trap "shutdown_servers; exit 0" INT - -test_whitelists -result=$? - -process_test_result $result $action_on_fail - -shutdown_servers - -sleep 2 - -test_blacklists -result=$? - -process_test_result $result $action_on_fail - -shutdown_servers - -exit $result - diff --git system_test/embedded_consumer/config/blacklisttest.consumer.properties system_test/embedded_consumer/config/blacklisttest.consumer.properties deleted file mode 100644 index 2d51bad..0000000 --- system_test/embedded_consumer/config/blacklisttest.consumer.properties +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.consumer.ConsumerConfig for more details - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -#consumer group id -groupid=group1 - -mirror.topics.blacklist=test02,test03 - diff --git system_test/embedded_consumer/config/consumer.properties system_test/embedded_consumer/config/consumer.properties deleted file mode 100644 index 8bcc48e..0000000 --- system_test/embedded_consumer/config/consumer.properties +++ /dev/null @@ -1,14 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git system_test/embedded_consumer/config/mirror_producer.properties system_test/embedded_consumer/config/mirror_producer.properties deleted file mode 100644 index 9ea68d0..0000000 --- system_test/embedded_consumer/config/mirror_producer.properties +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2182 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -producer.type=async - -# to avoid dropping events if the queue is full, wait indefinitely -queue.enqueueTimeout.ms=-1 - diff --git system_test/embedded_consumer/config/server_source1.properties system_test/embedded_consumer/config/server_source1.properties deleted file mode 100644 index 11bfe1d..0000000 --- system_test/embedded_consumer/config/server_source1.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -# the id of the broker -brokerid=1 - -# hostname of broker. If not set, will pick up from the value returned -# from getLocalHost. If there are multiple interfaces getLocalHost -# may not be what you want. -# hostname= - -# number of logical partitions on this broker -num.partitions=1 - -# the port the socket server runs on -port=9092 - -# the number of processor threads the socket server uses. Defaults to the number of cores on the machine -num.threads=8 - -# the directory in which to store log files -log.dir=/tmp/kafka-source1-logs - -# the send buffer used by the socket server -socket.send.buffer=1048576 - -# the receive buffer used by the socket server -socket.receive.buffer=1048576 - -# the maximum size of a log segment -log.file.size=10000000 - -# the interval between running cleanup on the logs -log.cleanup.interval.mins=1 - -# the minimum age of a log file to eligible for deletion -log.retention.hours=168 - -#the number of messages to accept without flushing the log to disk -log.flush.interval=600 - -#set the following properties to use zookeeper - -# enable connecting to zookeeper -enable.zookeeper=true - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -# time based topic flush intervals in ms -#topic.flush.intervals.ms=topic:1000 - -# default time based flush interval in ms -log.default.flush.interval.ms=1000 - -# time based topic flasher time rate in ms -log.default.flush.scheduler.interval.ms=1000 - diff --git system_test/embedded_consumer/config/server_source2.properties system_test/embedded_consumer/config/server_source2.properties deleted file mode 100644 index 24991bb..0000000 --- system_test/embedded_consumer/config/server_source2.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -# the id of the broker -brokerid=2 - -# hostname of broker. If not set, will pick up from the value returned -# from getLocalHost. If there are multiple interfaces getLocalHost -# may not be what you want. -# hostname= - -# number of logical partitions on this broker -num.partitions=1 - -# the port the socket server runs on -port=9091 - -# the number of processor threads the socket server uses. Defaults to the number of cores on the machine -num.threads=8 - -# the directory in which to store log files -log.dir=/tmp/kafka-source2-logs - -# the send buffer used by the socket server -socket.send.buffer=1048576 - -# the receive buffer used by the socket server -socket.receive.buffer=1048576 - -# the maximum size of a log segment -log.file.size=536870912 - -# the interval between running cleanup on the logs -log.cleanup.interval.mins=1 - -# the minimum age of a log file to eligible for deletion -log.retention.hours=168 - -#the number of messages to accept without flushing the log to disk -log.flush.interval=600 - -#set the following properties to use zookeeper - -# enable connecting to zookeeper -enable.zookeeper=true - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -# time based topic flush intervals in ms -#topic.flush.intervals.ms=topic:1000 - -# default time based flush interval in ms -log.default.flush.interval.ms=1000 - -# time based topic flasher time rate in ms -log.default.flush.scheduler.interval.ms=1000 - diff --git system_test/embedded_consumer/config/server_source3.properties system_test/embedded_consumer/config/server_source3.properties deleted file mode 100644 index 02c2cde..0000000 --- system_test/embedded_consumer/config/server_source3.properties +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -# the id of the broker -brokerid=3 - -# hostname of broker. If not set, will pick up from the value returned -# from getLocalHost. If there are multiple interfaces getLocalHost -# may not be what you want. -# hostname= - -# number of logical partitions on this broker -num.partitions=1 - -# the port the socket server runs on -port=9090 - -# the number of processor threads the socket server uses. Defaults to the number of cores on the machine -num.threads=8 - -# the directory in which to store log files -log.dir=/tmp/kafka-source3-logs - -# the send buffer used by the socket server -socket.send.buffer=1048576 - -# the receive buffer used by the socket server -socket.receive.buffer=1048576 - -# the maximum size of a log segment -log.file.size=536870912 - -# the interval between running cleanup on the logs -log.cleanup.interval.mins=1 - -# the minimum age of a log file to eligible for deletion -log.retention.hours=168 - -#the number of messages to accept without flushing the log to disk -log.flush.interval=600 - -#set the following properties to use zookeeper - -# enable connecting to zookeeper -enable.zookeeper=true - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -# time based topic flush intervals in ms -#topic.flush.intervals.ms=topic:1000 - -# default time based flush interval in ms -log.default.flush.interval.ms=1000 - -# time based topic flasher time rate in ms -log.default.flush.scheduler.interval.ms=1000 - diff --git system_test/embedded_consumer/config/server_target1.properties system_test/embedded_consumer/config/server_target1.properties deleted file mode 100644 index 72da002..0000000 --- system_test/embedded_consumer/config/server_target1.properties +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -# the id of the broker -brokerid=1 - -# hostname of broker. If not set, will pick up from the value returned -# from getLocalHost. If there are multiple interfaces getLocalHost -# may not be what you want. -# hostname= - -# number of logical partitions on this broker -num.partitions=1 - -# the port the socket server runs on -port=9093 - -# the number of processor threads the socket server uses. Defaults to the number of cores on the machine -num.threads=8 - -# the directory in which to store log files -log.dir=/tmp/kafka-target1-logs - -# the send buffer used by the socket server -socket.send.buffer=1048576 - -# the receive buffer used by the socket server -socket.receive.buffer=1048576 - -# the maximum size of a log segment -log.file.size=536870912 - -# the interval between running cleanup on the logs -log.cleanup.interval.mins=1 - -# the minimum age of a log file to eligible for deletion -log.retention.hours=168 - -#the number of messages to accept without flushing the log to disk -log.flush.interval=600 - -#set the following properties to use zookeeper - -# enable connecting to zookeeper -enable.zookeeper=true - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2182 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -# time based topic flush intervals in ms -#topic.flush.intervals.ms=topic:1000 - -# default time based flush interval in ms -log.default.flush.interval.ms=1000 - -# time based topic flasher time rate in ms -log.default.flush.scheduler.interval.ms=1000 - -# topic partition count map -# topic.partition.count.map=topic1:3, topic2:4 diff --git system_test/embedded_consumer/config/server_target2.properties system_test/embedded_consumer/config/server_target2.properties deleted file mode 100644 index 96c5295..0000000 --- system_test/embedded_consumer/config/server_target2.properties +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.server.KafkaConfig for additional details and defaults - -# the id of the broker -brokerid=2 - -# hostname of broker. If not set, will pick up from the value returned -# from getLocalHost. If there are multiple interfaces getLocalHost -# may not be what you want. -# hostname= - -# number of logical partitions on this broker -num.partitions=1 - -# the port the socket server runs on -port=9094 - -# the number of processor threads the socket server uses. Defaults to the number of cores on the machine -num.threads=8 - -# the directory in which to store log files -log.dir=/tmp/kafka-target2-logs - -# the send buffer used by the socket server -socket.send.buffer=1048576 - -# the receive buffer used by the socket server -socket.receive.buffer=1048576 - -# the maximum size of a log segment -log.file.size=536870912 - -# the interval between running cleanup on the logs -log.cleanup.interval.mins=1 - -# the minimum age of a log file to eligible for deletion -log.retention.hours=168 - -#the number of messages to accept without flushing the log to disk -log.flush.interval=600 - -#set the following properties to use zookeeper - -# enable connecting to zookeeper -enable.zookeeper=true - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2182 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -# time based topic flush intervals in ms -#topic.flush.intervals.ms=topic:1000 - -# default time based flush interval in ms -log.default.flush.interval.ms=1000 - -# time based topic flasher time rate in ms -log.default.flush.scheduler.interval.ms=1000 - -# topic partition count map -# topic.partition.count.map=topic1:3, topic2:4 diff --git system_test/embedded_consumer/config/whitelisttest.consumer.properties system_test/embedded_consumer/config/whitelisttest.consumer.properties deleted file mode 100644 index 5ff54ba..0000000 --- system_test/embedded_consumer/config/whitelisttest.consumer.properties +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.consumer.ConsumerConfig for more details - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=localhost:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -#consumer group id -groupid=group1 - -mirror.topics.whitelist=test01 - diff --git system_test/embedded_consumer/config/zookeeper_source.properties system_test/embedded_consumer/config/zookeeper_source.properties deleted file mode 100644 index 76b02a2..0000000 --- system_test/embedded_consumer/config/zookeeper_source.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# the directory where the snapshot is stored. -dataDir=/tmp/zookeeper_source -# the port at which the clients will connect -clientPort=2181 diff --git system_test/embedded_consumer/config/zookeeper_target.properties system_test/embedded_consumer/config/zookeeper_target.properties deleted file mode 100644 index 28561d9..0000000 --- system_test/embedded_consumer/config/zookeeper_target.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# the directory where the snapshot is stored. -dataDir=/tmp/zookeeper_target -# the port at which the clients will connect -clientPort=2182 diff --git system_test/embedded_consumer/expected.out system_test/embedded_consumer/expected.out deleted file mode 100644 index 2d64ec9..0000000 --- system_test/embedded_consumer/expected.out +++ /dev/null @@ -1,11 +0,0 @@ -start the servers ... -start producing messages ... -Total Num Messages: 10000000 bytes: 1994374785 in 106.076 secs -Messages/sec: 94272.0314 -MB/sec: 17.9304 -[2011-05-02 11:50:29,022] INFO Disconnecting from localhost:9092 (kafka.producer.SyncProducer) -wait for consumer to finish consuming ... -test passed -bin/../../../bin/kafka-server-start.sh: line 11: 359 Terminated $(dirname $0)/kafka-run-class.sh kafka.Kafka $@ -bin/../../../bin/zookeeper-server-start.sh: line 9: 357 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ -bin/../../../bin/zookeeper-server-start.sh: line 9: 358 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ diff --git system_test/mirror_maker/README system_test/mirror_maker/README new file mode 100644 index 0000000..da53c14 --- /dev/null +++ system_test/mirror_maker/README @@ -0,0 +1,22 @@ +This test replicates messages from two source kafka clusters into one target +kafka cluster using the mirror-maker tool. At the end, the messages produced +at the source brokers should match that at the target brokers. + +To run this test, do +bin/run-test.sh + +In the event of failure, by default the brokers and zookeepers remain running +to make it easier to debug the issue - hit Ctrl-C to shut them down. You can +change this behavior by setting the action_on_fail flag in the script to "exit" +or "proceed", in which case a snapshot of all the logs and directories is +placed in the test's base directory. + +It is a good idea to run the test in a loop. E.g.: + +:>/tmp/mirrormaker_test.log +for i in {1..10}; do echo "run $i"; ./bin/run-test.sh 2>1 >> /tmp/mirrormaker_test.log; done +tail -F /tmp/mirrormaker_test.log + +grep -ic passed /tmp/mirrormaker_test.log +grep -ic failed /tmp/mirrormaker_test.log + diff --git system_test/mirror_maker/bin/expected.out system_test/mirror_maker/bin/expected.out new file mode 100644 index 0000000..0a1bbaf --- /dev/null +++ system_test/mirror_maker/bin/expected.out @@ -0,0 +1,18 @@ +start the servers ... +start producing messages ... +wait for consumer to finish consuming ... +[2011-05-17 14:49:11,605] INFO Creating async producer for broker id = 2 at localhost:9091 (kafka.producer.ProducerPool) +[2011-05-17 14:49:11,606] INFO Creating async producer for broker id = 1 at localhost:9092 (kafka.producer.ProducerPool) +[2011-05-17 14:49:11,607] INFO Creating async producer for broker id = 3 at localhost:9090 (kafka.producer.ProducerPool) +thread 0: 400000 messages sent 3514012.1233 nMsg/sec 3.3453 MBs/sec +[2011-05-17 14:49:34,382] INFO Closing all async producers (kafka.producer.ProducerPool) +[2011-05-17 14:49:34,383] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) +[2011-05-17 14:49:34,384] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) +[2011-05-17 14:49:34,385] INFO Closed AsyncProducer (kafka.producer.async.AsyncProducer) +Total Num Messages: 400000 bytes: 79859641 in 22.93 secs +Messages/sec: 17444.3960 +MB/sec: 3.3214 +test passed +stopping the servers +bin/../../../bin/zookeeper-server-start.sh: line 9: 22584 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ +bin/../../../bin/zookeeper-server-start.sh: line 9: 22585 Terminated $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ diff --git system_test/mirror_maker/bin/run-test.sh system_test/mirror_maker/bin/run-test.sh new file mode 100755 index 0000000..bdc3f37 --- /dev/null +++ system_test/mirror_maker/bin/run-test.sh @@ -0,0 +1,357 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +readonly num_messages=10000 +readonly message_size=100 +readonly action_on_fail="proceed" +# readonly action_on_fail="wait" + +readonly test_start_time="$(date +%s)" + +readonly base_dir=$(dirname $0)/.. + +info() { + echo -e "$(date +"%Y-%m-%d %H:%M:%S") $*" +} + +kill_child_processes() { + isTopmost=$1 + curPid=$2 + childPids=$(ps a -o pid= -o ppid= | grep "${curPid}$" | awk '{print $1;}') + for childPid in $childPids + do + kill_child_processes 0 $childPid + done + if [ $isTopmost -eq 0 ]; then + kill -15 $curPid 2> /dev/null + fi +} + +cleanup() { + info "cleaning up" + + pid_zk_source1= + pid_zk_source2= + pid_zk_target= + pid_kafka_source_1_1= + pid_kafka_source_1_2= + pid_kafka_source_2_1= + pid_kafka_source_2_2= + pid_kafka_target_1_1= + pid_kafka_target_1_2= + pid_producer= + pid_mirrormaker_1= + pid_mirrormaker_2= + + rm -rf /tmp/zookeeper* + + rm -rf /tmp/kafka* +} + +begin_timer() { + t_begin=$(date +%s) +} + +end_timer() { + t_end=$(date +%s) +} + +start_zk() { + info "starting zookeepers" + $base_dir/../../bin/zookeeper-server-start.sh $base_dir/config/zookeeper_source_1.properties 2>&1 > $base_dir/zookeeper_source-1.log & + pid_zk_source1=$! + $base_dir/../../bin/zookeeper-server-start.sh $base_dir/config/zookeeper_source_2.properties 2>&1 > $base_dir/zookeeper_source-2.log & + pid_zk_source2=$! + $base_dir/../../bin/zookeeper-server-start.sh $base_dir/config/zookeeper_target.properties 2>&1 > $base_dir/zookeeper_target.log & + pid_zk_target=$! +} + +start_source_servers() { + info "starting source cluster" + + JMX_PORT=1111 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source_1_1.properties 2>&1 > $base_dir/kafka_source-1-1.log & + pid_kafka_source_1_1=$! + JMX_PORT=2222 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source_1_2.properties 2>&1 > $base_dir/kafka_source-1-2.log & + pid_kafka_source_1_2=$! + JMX_PORT=3333 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source_2_1.properties 2>&1 > $base_dir/kafka_source-2-1.log & + pid_kafka_source_2_1=$! + JMX_PORT=4444 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_source_2_2.properties 2>&1 > $base_dir/kafka_source-2-2.log & + pid_kafka_source_2_2=$! +} + +start_target_servers() { + info "starting mirror cluster" + JMX_PORT=5555 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target_1_1.properties 2>&1 > $base_dir/kafka_target-1-1.log & + pid_kafka_target_1_1=$! + JMX_PORT=6666 $base_dir/../../bin/kafka-run-class.sh kafka.Kafka $base_dir/config/server_target_1_2.properties 2>&1 > $base_dir/kafka_target-1-2.log & + pid_kafka_target_1_2=$! +} + +shutdown_servers() { + info "stopping mirror-maker" + if [ "x${pid_mirrormaker_1}" != "x" ]; then kill_child_processes 0 ${pid_mirrormaker_1}; fi + # sleep to avoid rebalancing during shutdown + sleep 2 + if [ "x${pid_mirrormaker_2}" != "x" ]; then kill_child_processes 0 ${pid_mirrormaker_2}; fi + + info "stopping producer" + if [ "x${pid_producer}" != "x" ]; then kill_child_processes 0 ${pid_producer}; fi + + info "shutting down target servers" + if [ "x${pid_kafka_target_1_1}" != "x" ]; then kill_child_processes 0 ${pid_kafka_target_1_1}; fi + if [ "x${pid_kafka_target_1_2}" != "x" ]; then kill_child_processes 0 ${pid_kafka_target_1_2}; fi + sleep 2 + + info "shutting down source servers" + if [ "x${pid_kafka_source_1_1}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source_1_1}; fi + if [ "x${pid_kafka_source_1_2}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source_1_2}; fi + if [ "x${pid_kafka_source_2_1}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source_2_1}; fi + if [ "x${pid_kafka_source_2_2}" != "x" ]; then kill_child_processes 0 ${pid_kafka_source_2_2}; fi + + info "shutting down zookeeper servers" + if [ "x${pid_zk_target}" != "x" ]; then kill_child_processes 0 ${pid_zk_target}; fi + if [ "x${pid_zk_source1}" != "x" ]; then kill_child_processes 0 ${pid_zk_source1}; fi + if [ "x${pid_zk_source2}" != "x" ]; then kill_child_processes 0 ${pid_zk_source2}; fi +} + +start_producer() { + topic=$1 + zk=$2 + info "start producing messages for topic $topic to zookeeper $zk ..." + $base_dir/../../bin/kafka-run-class.sh kafka.perf.ProducerPerformance --brokerinfo zk.connect=$zk --topic $topic --messages $num_messages --message-size $message_size --batch-size 200 --vary-message-size --threads 1 --reporting-interval $num_messages --async 2>&1 > $base_dir/producer_performance.log & + pid_producer=$! +} + +# Usage: wait_partition_done ([kafka-server] [topic] [partition-id])+ +wait_partition_done() { + n_tuples=$(($# / 3)) + + i=1 + while (($#)); do + kafka_server[i]=$1 + topic[i]=$2 + partitionid[i]=$3 + prev_offset[i]=0 + info "\twaiting for partition on server ${kafka_server[i]}, topic ${topic[i]}, partition ${partitionid[i]}" + i=$((i+1)) + shift 3 + done + + all_done=0 + + # set -x + while [[ $all_done != 1 ]]; do + sleep 4 + i=$n_tuples + all_done=1 + for ((i=1; i <= $n_tuples; i++)); do + cur_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server ${kafka_server[i]} --topic ${topic[i]} --partition ${partitionid[i]} --time -1 --offsets 1 | tail -1) + if [ "x$cur_size" != "x${prev_offset[i]}" ]; then + all_done=0 + prev_offset[i]=$cur_size + fi + done + done + +} + +cmp_logs() { + topic=$1 + info "comparing source and target logs for topic $topic" + source_part0_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9090 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + source_part1_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9091 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + source_part2_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9092 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + source_part3_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9093 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + target_part0_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9094 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + target_part1_size=$($base_dir/../../bin/kafka-run-class.sh kafka.tools.GetOffsetShell --server kafka://localhost:9095 --topic $topic --partition 0 --time -1 --offsets 1 | tail -1) + if [ "x$source_part0_size" == "x" ]; then source_part0_size=0; fi + if [ "x$source_part1_size" == "x" ]; then source_part1_size=0; fi + if [ "x$source_part2_size" == "x" ]; then source_part2_size=0; fi + if [ "x$source_part3_size" == "x" ]; then source_part3_size=0; fi + if [ "x$target_part0_size" == "x" ]; then target_part0_size=0; fi + if [ "x$target_part1_size" == "x" ]; then target_part1_size=0; fi + expected_size=$(($source_part0_size + $source_part1_size + $source_part2_size + $source_part3_size)) + actual_size=$(($target_part0_size + $target_part1_size)) + if [ "x$expected_size" != "x$actual_size" ] + then + info "source size: $expected_size target size: $actual_size" + return 1 + else + return 0 + fi +} + +take_fail_snapshot() { + snapshot_dir="$base_dir/failed-${snapshot_prefix}-${test_start_time}" + mkdir $snapshot_dir + for dir in /tmp/zookeeper_source{1..2} /tmp/zookeeper_target /tmp/kafka-source-{1..2}-{1..2}-logs /tmp/kafka-target{1..2}-logs; do + if [ -d $dir ]; then + cp -r $dir $snapshot_dir + fi + done +} + +# Usage: process_test_result +# result: last test result +# action_on_fail: (exit|wait|proceed) +# ("wait" is useful if you want to troubleshoot using zookeeper) +process_test_result() { + result=$1 + if [ $1 -eq 0 ]; then + info "test passed" + else + info "test failed" + case "$2" in + "wait") info "waiting: hit Ctrl-c to quit" + wait + ;; + "exit") shutdown_servers + take_fail_snapshot + exit $result + ;; + *) shutdown_servers + take_fail_snapshot + info "proceeding" + ;; + esac + fi +} + +test_whitelists() { + info "### Testing whitelists" + snapshot_prefix="whitelist-test" + + cleanup + start_zk + start_source_servers + start_target_servers + sleep 4 + + info "starting mirror makers" + JMX_PORT=7777 $base_dir/../../bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer-config $base_dir/config/whitelisttest_1.consumer.properties --consumer-config $base_dir/config/whitelisttest_2.consumer.properties --producer-config $base_dir/config/mirror_producer.properties --whitelist="white.*" --num-streams 2 2>&1 > $base_dir/kafka_mirrormaker_1.log & + pid_mirrormaker_1=$! + JMX_PORT=8888 $base_dir/../../bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer-config $base_dir/config/whitelisttest_1.consumer.properties --consumer-config $base_dir/config/whitelisttest_2.consumer.properties --producer-config $base_dir/config/mirror_producer.properties --whitelist="white.*" --num-streams 2 2>&1 > $base_dir/kafka_mirrormaker_2.log & + pid_mirrormaker_2=$! + + begin_timer + + start_producer whitetopic01 localhost:2181 + start_producer whitetopic01 localhost:2182 + info "waiting for whitetopic01 producers to finish producing ..." + wait_partition_done kafka://localhost:9090 whitetopic01 0 kafka://localhost:9091 whitetopic01 0 kafka://localhost:9092 whitetopic01 0 kafka://localhost:9093 whitetopic01 0 + + start_producer whitetopic02 localhost:2181 + start_producer whitetopic03 localhost:2181 + start_producer whitetopic04 localhost:2182 + info "waiting for whitetopic02,whitetopic03,whitetopic04 producers to finish producing ..." + wait_partition_done kafka://localhost:9090 whitetopic02 0 kafka://localhost:9091 whitetopic02 0 kafka://localhost:9090 whitetopic03 0 kafka://localhost:9091 whitetopic03 0 kafka://localhost:9092 whitetopic04 0 kafka://localhost:9093 whitetopic04 0 + + start_producer blacktopic01 localhost:2182 + info "waiting for blacktopic01 producer to finish producing ..." + wait_partition_done kafka://localhost:9092 blacktopic01 0 kafka://localhost:9093 blacktopic01 0 + + info "waiting for consumer to finish consuming ..." + + wait_partition_done kafka://localhost:9094 whitetopic01 0 kafka://localhost:9095 whitetopic01 0 kafka://localhost:9094 whitetopic02 0 kafka://localhost:9095 whitetopic02 0 kafka://localhost:9094 whitetopic03 0 kafka://localhost:9095 whitetopic03 0 kafka://localhost:9094 whitetopic04 0 kafka://localhost:9095 whitetopic04 0 + + end_timer + info "embedded consumer took $((t_end - t_begin)) seconds" + + sleep 2 + + # if [[ -d /tmp/kafka-target-1-1-logs/blacktopic01 || /tmp/kafka-target-1-2-logs/blacktopic01 ]]; then + # echo "blacktopic01 found on target cluster" + # result=1 + # else + # cmp_logs whitetopic01 && cmp_logs whitetopic02 && cmp_logs whitetopic03 && cmp_logs whitetopic04 + # result=$? + # fi + + cmp_logs blacktopic01 + + cmp_logs whitetopic01 && cmp_logs whitetopic02 && cmp_logs whitetopic03 && cmp_logs whitetopic04 + result=$? + + return $result +} + +test_blacklists() { + info "### Testing blacklists" + snapshot_prefix="blacklist-test" + cleanup + start_zk + start_source_servers + start_target_servers + sleep 4 + + info "starting mirror maker" + $base_dir/../../bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer-config $base_dir/config/blacklisttest.consumer.properties --producer-config $base_dir/config/mirror_producer.properties --blacklist="black.*" --num-streams 2 2>&1 > $base_dir/kafka_mirrormaker_1.log & + pid_mirrormaker_1=$! + + start_producer blacktopic01 localhost:2181 + start_producer blacktopic02 localhost:2181 + info "waiting for producer to finish producing blacktopic01,blacktopic02 ..." + wait_partition_done kafka://localhost:9090 blacktopic01 0 kafka://localhost:9091 blacktopic01 0 kafka://localhost:9090 blacktopic02 0 kafka://localhost:9091 blacktopic02 0 + + begin_timer + + start_producer whitetopic01 localhost:2181 + info "waiting for producer to finish producing whitetopic01 ..." + wait_partition_done kafka://localhost:9090 whitetopic01 0 kafka://localhost:9091 whitetopic01 0 + + info "waiting for consumer to finish consuming ..." + wait_partition_done kafka://localhost:9094 whitetopic01 0 kafka://localhost:9095 whitetopic01 0 + + end_timer + + info "embedded consumer took $((t_end - t_begin)) seconds" + + sleep 2 + + cmp_logs blacktopic01 || cmp_logs blacktopic02 + if [ $? -eq 0 ]; then + return 1 + fi + + cmp_logs whitetopic01 + return $? +} + +# main test begins + +echo "Test-$test_start_time" + +# Ctrl-c trap. Catches INT signal +trap "shutdown_servers; exit 0" INT + +test_whitelists +result=$? + +process_test_result $result $action_on_fail + +shutdown_servers + +sleep 2 + +test_blacklists +result=$? + +process_test_result $result $action_on_fail + +shutdown_servers + +exit $result + diff --git system_test/mirror_maker/config/blacklisttest.consumer.properties system_test/mirror_maker/config/blacklisttest.consumer.properties new file mode 100644 index 0000000..6ea85ec --- /dev/null +++ system_test/mirror_maker/config/blacklisttest.consumer.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2181 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +#consumer group id +groupid=group1 +shallowiterator.enable=true + diff --git system_test/mirror_maker/config/mirror_producer.properties system_test/mirror_maker/config/mirror_producer.properties new file mode 100644 index 0000000..5940c24 --- /dev/null +++ system_test/mirror_maker/config/mirror_producer.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2183 +# broker.list=1:localhost:9094,2:localhost:9095 + +# timeout in ms for connecting to zookeeper +# zk.connectiontimeout.ms=1000000 + +producer.type=async + +# to avoid dropping events if the queue is full, wait indefinitely +queue.enqueueTimeout.ms=-1 + diff --git system_test/mirror_maker/config/server_source_1_1.properties system_test/mirror_maker/config/server_source_1_1.properties new file mode 100644 index 0000000..d89c4fb --- /dev/null +++ system_test/mirror_maker/config/server_source_1_1.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=1 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9090 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-source-1-1-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=10000000 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2181 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + diff --git system_test/mirror_maker/config/server_source_1_2.properties system_test/mirror_maker/config/server_source_1_2.properties new file mode 100644 index 0000000..063d68b --- /dev/null +++ system_test/mirror_maker/config/server_source_1_2.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=2 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9091 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-source-1-2-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=536870912 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2181 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + diff --git system_test/mirror_maker/config/server_source_2_1.properties system_test/mirror_maker/config/server_source_2_1.properties new file mode 100644 index 0000000..998b460 --- /dev/null +++ system_test/mirror_maker/config/server_source_2_1.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=1 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9092 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-source-2-1-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=536870912 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2182 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + diff --git system_test/mirror_maker/config/server_source_2_2.properties system_test/mirror_maker/config/server_source_2_2.properties new file mode 100644 index 0000000..81427ae --- /dev/null +++ system_test/mirror_maker/config/server_source_2_2.properties @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=2 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9093 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-source-2-2-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=536870912 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2182 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + diff --git system_test/mirror_maker/config/server_target_1_1.properties system_test/mirror_maker/config/server_target_1_1.properties new file mode 100644 index 0000000..0265f4e --- /dev/null +++ system_test/mirror_maker/config/server_target_1_1.properties @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=1 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9094 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-target-1-1-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=536870912 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2183 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + +# topic partition count map +# topic.partition.count.map=topic1:3, topic2:4 diff --git system_test/mirror_maker/config/server_target_1_2.properties system_test/mirror_maker/config/server_target_1_2.properties new file mode 100644 index 0000000..a31e9ca --- /dev/null +++ system_test/mirror_maker/config/server_target_1_2.properties @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +# the id of the broker +brokerid=2 + +# hostname of broker. If not set, will pick up from the value returned +# from getLocalHost. If there are multiple interfaces getLocalHost +# may not be what you want. +# hostname= + +# number of logical partitions on this broker +num.partitions=1 + +# the port the socket server runs on +port=9095 + +# the number of processor threads the socket server uses. Defaults to the number of cores on the machine +num.threads=8 + +# the directory in which to store log files +log.dir=/tmp/kafka-target-1-2-logs + +# the send buffer used by the socket server +socket.send.buffer=1048576 + +# the receive buffer used by the socket server +socket.receive.buffer=1048576 + +# the maximum size of a log segment +log.file.size=536870912 + +# the interval between running cleanup on the logs +log.cleanup.interval.mins=1 + +# the minimum age of a log file to eligible for deletion +log.retention.hours=168 + +#the number of messages to accept without flushing the log to disk +log.flush.interval=600 + +#set the following properties to use zookeeper + +# enable connecting to zookeeper +enable.zookeeper=true + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2183 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +# time based topic flush intervals in ms +#topic.flush.intervals.ms=topic:1000 + +# default time based flush interval in ms +log.default.flush.interval.ms=1000 + +# time based topic flasher time rate in ms +log.default.flush.scheduler.interval.ms=1000 + +# topic partition count map +# topic.partition.count.map=topic1:3, topic2:4 diff --git system_test/mirror_maker/config/whitelisttest_1.consumer.properties system_test/mirror_maker/config/whitelisttest_1.consumer.properties new file mode 100644 index 0000000..6ea85ec --- /dev/null +++ system_test/mirror_maker/config/whitelisttest_1.consumer.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2181 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +#consumer group id +groupid=group1 +shallowiterator.enable=true + diff --git system_test/mirror_maker/config/whitelisttest_2.consumer.properties system_test/mirror_maker/config/whitelisttest_2.consumer.properties new file mode 100644 index 0000000..e11112f --- /dev/null +++ system_test/mirror_maker/config/whitelisttest_2.consumer.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=localhost:2182 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +#consumer group id +groupid=group1 +shallowiterator.enable=true + diff --git system_test/mirror_maker/config/zookeeper_source_1.properties system_test/mirror_maker/config/zookeeper_source_1.properties new file mode 100644 index 0000000..f851796 --- /dev/null +++ system_test/mirror_maker/config/zookeeper_source_1.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper_source-1 +# the port at which the clients will connect +clientPort=2181 diff --git system_test/mirror_maker/config/zookeeper_source_2.properties system_test/mirror_maker/config/zookeeper_source_2.properties new file mode 100644 index 0000000..d534d18 --- /dev/null +++ system_test/mirror_maker/config/zookeeper_source_2.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper_source-2 +# the port at which the clients will connect +clientPort=2182 diff --git system_test/mirror_maker/config/zookeeper_target.properties system_test/mirror_maker/config/zookeeper_target.properties new file mode 100644 index 0000000..55a7eb1 --- /dev/null +++ system_test/mirror_maker/config/zookeeper_target.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper_target +# the port at which the clients will connect +clientPort=2183