Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerTests.cs (working copy) @@ -23,7 +23,6 @@ using System.Reflection; using System.Text; using System.Threading; - using Kafka.Client.Cfg; using Kafka.Client.Consumers; using Kafka.Client.Exceptions; using Kafka.Client.Messages; @@ -34,21 +33,10 @@ [TestFixture] public class ConsumerTests : IntegrationFixtureBase { - /// - /// Kafka Client configuration - /// - private static KafkaClientConfiguration clientConfig; - - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - [Test] public void ConsumerConnectorIsCreatedConnectsDisconnectsAndShutsDown() { - var config = new ConsumerConfig(clientConfig); + var config = this.ZooKeeperBasedConsumerConfig; using (new ZookeeperConsumerConnector(config, true)) { } @@ -57,6 +45,9 @@ [Test] public void SimpleSyncProducerSends2MessagesAndConsumerConnectorGetsThemBack() { + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ZooKeeperBasedConsumerConfig; + // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); @@ -66,15 +57,15 @@ byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); - var producerConfig = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(producerConfig); var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { msg1, msg2 }); - producer.Send(producerRequest); + using (var producer = new SyncProducer(prodConfig)) + { + producer.Send(producerRequest); + } // now consuming - var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List(); - using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) + using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true)) { var topicCount = new Dictionary { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); @@ -103,53 +94,57 @@ [Test] public void OneMessageIsSentAndReceivedThenExceptionsWhenNoMessageThenAnotherMessageIsSentAndReceived() { + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ZooKeeperBasedConsumerConfig; + // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); var msg1 = new Message(payloadData1); - - var producerConfig = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(producerConfig); - var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { msg1 }); - producer.Send(producerRequest); - - // now consuming - var config = new ConsumerConfig(clientConfig) { AutoCommit = false, Timeout = 5000 }; - using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) + using (var producer = new SyncProducer(prodConfig)) { - var topicCount = new Dictionary { { CurrentTestTopic, 1 } }; - var messages = consumerConnector.CreateMessageStreams(topicCount); - var sets = messages[CurrentTestTopic]; - KafkaMessageStream myStream = sets[0]; - var enumerator = myStream.GetEnumerator(); + var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { msg1 }); + producer.Send(producerRequest); - Assert.IsTrue(enumerator.MoveNext()); - Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); + // now consuming + using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true)) + { + var topicCount = new Dictionary { { CurrentTestTopic, 1 } }; + var messages = consumerConnector.CreateMessageStreams(topicCount); + var sets = messages[CurrentTestTopic]; + KafkaMessageStream myStream = sets[0]; + var enumerator = myStream.GetEnumerator(); - Assert.Throws(() => enumerator.MoveNext()); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual(msg1.ToString(), enumerator.Current.ToString()); - Assert.Throws(() => enumerator.MoveNext()); // iterator is in failed state + Assert.Throws(() => enumerator.MoveNext()); - enumerator.Reset(); + Assert.Throws(() => enumerator.MoveNext()); // iterator is in failed state - // producing again - string payload2 = "kafka 2."; - byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); - var msg2 = new Message(payloadData2); + enumerator.Reset(); - var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List { msg2 }); - producer.Send(producerRequest2); + // producing again + string payload2 = "kafka 2."; + byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); + var msg2 = new Message(payloadData2); - Thread.Sleep(3000); + var producerRequest2 = new ProducerRequest(CurrentTestTopic, 0, new List { msg2 }); + producer.Send(producerRequest2); + Thread.Sleep(3000); - Assert.IsTrue(enumerator.MoveNext()); - Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); + Assert.IsTrue(enumerator.MoveNext()); + Assert.AreEqual(msg2.ToString(), enumerator.Current.ToString()); + } } } [Test] public void ConsumerConnectorConsumesTwoDifferentTopics() { + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ZooKeeperBasedConsumerConfig; + string topic1 = CurrentTestTopic + "1"; string topic2 = CurrentTestTopic + "2"; @@ -162,18 +157,18 @@ byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); - var producerConfig = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(producerConfig); - var producerRequest1 = new ProducerRequest(topic1, 0, new List { msg1 }); - producer.Send(producerRequest1); - var producerRequest2 = new ProducerRequest(topic2, 0, new List { msg2 }); - producer.Send(producerRequest2); + using (var producer = new SyncProducer(prodConfig)) + { + var producerRequest1 = new ProducerRequest(topic1, 0, new List { msg1 }); + producer.Send(producerRequest1); + var producerRequest2 = new ProducerRequest(topic2, 0, new List { msg2 }); + producer.Send(producerRequest2); + } // now consuming - var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages1 = new List(); var resultMessages2 = new List(); - using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) + using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true)) { var topicCount = new Dictionary { { topic1, 1 }, { topic2, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); @@ -224,9 +219,10 @@ [Test] public void ConsumerConnectorReceivesAShutdownSignal() { + var consumerConfig = this.ZooKeeperBasedConsumerConfig; + // now consuming - var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; - using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) + using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true)) { var topicCount = new Dictionary { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); @@ -260,6 +256,9 @@ [Test] public void ProducersSendMessagesToDifferentPartitionsAndConsumerConnectorGetsThemBack() { + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ZooKeeperBasedConsumerConfig; + // first producing string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); @@ -269,23 +268,21 @@ byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); var msg2 = new Message(payloadData2); - var producerConfig = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(producerConfig); - var producerRequest1 = new ProducerRequest(CurrentTestTopic, 0, new List() { msg1 }); - producer.Send(producerRequest1); - var producerRequest2 = new ProducerRequest(CurrentTestTopic, 1, new List() { msg2 }); - producer.Send(producerRequest2); + using (var producer = new SyncProducer(prodConfig)) + { + var producerRequest1 = new ProducerRequest(CurrentTestTopic, 0, new List { msg1 }); + producer.Send(producerRequest1); + var producerRequest2 = new ProducerRequest(CurrentTestTopic, 1, new List { msg2 }); + producer.Send(producerRequest2); + } // now consuming - var config = new ConsumerConfig(clientConfig) { AutoCommit = false }; var resultMessages = new List(); - using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(config, true)) + using (IConsumerConnector consumerConnector = new ZookeeperConsumerConnector(consumerConfig, true)) { var topicCount = new Dictionary { { CurrentTestTopic, 1 } }; var messages = consumerConnector.CreateMessageStreams(topicCount); - var sets = messages[CurrentTestTopic]; - try { foreach (var set in sets) Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestHelper.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestHelper.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestHelper.cs (working copy) @@ -19,33 +19,30 @@ { using System; using System.Collections.Generic; + using System.Linq; using Kafka.Client.Cfg; using Kafka.Client.Consumers; using Kafka.Client.Requests; public static class TestHelper { - public static long GetCurrentKafkaOffset(string topic, KafkaClientConfiguration clientConfig) + public static long GetCurrentKafkaOffset(string topic, ConsumerConfiguration clientConfig) { - return GetCurrentKafkaOffset(topic, clientConfig.KafkaServer.Address, clientConfig.KafkaServer.Port); + return GetCurrentKafkaOffset(topic, clientConfig.Broker.Host, clientConfig.Broker.Port); } public static long GetCurrentKafkaOffset(string topic, string address, int port) { - OffsetRequest request = new OffsetRequest(topic, 0, DateTime.Now.AddDays(-5).Ticks, 10); - ConsumerConfig consumerConfig = new ConsumerConfig(); - consumerConfig.Host = address; - consumerConfig.Port = port; - IConsumer consumer = new Consumers.Consumer(consumerConfig); + return GetCurrentKafkaOffset(topic, address, port, 0); + } + + public static long GetCurrentKafkaOffset(string topic, string address, int port, int partition) + { + var request = new OffsetRequest(topic, partition, DateTime.Now.AddDays(-5).Ticks, 10); + var consumerConfig = new ConsumerConfiguration(address, port); + IConsumer consumer = new Consumer(consumerConfig, address, port); IList list = consumer.GetOffsetsBefore(request); - if (list.Count > 0) - { - return list[0]; - } - else - { - return 0; - } + return list.Sum(); } } } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/KafkaIntegrationTest.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/KafkaIntegrationTest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/KafkaIntegrationTest.cs (working copy) @@ -22,7 +22,6 @@ using System.Linq; using System.Text; using System.Threading; - using Kafka.Client.Cfg; using Kafka.Client.Consumers; using Kafka.Client.Messages; using Kafka.Client.Producers.Async; @@ -37,39 +36,31 @@ public class KafkaIntegrationTest : IntegrationFixtureBase { /// - /// Kafka Client configuration - /// - private static KafkaClientConfiguration clientConfig; - - /// /// Maximum amount of time to wait trying to get a specific test message from Kafka server (in miliseconds) /// private static readonly int MaxTestWaitTimeInMiliseconds = 5000; - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - /// /// Sends a pair of message to Kafka. /// [Test] public void ProducerSendsMessage() { + var prodConfig = this.SyncProducerConfig1; + string payload1 = "kafka 1."; byte[] payloadData1 = Encoding.UTF8.GetBytes(payload1); - Message msg1 = new Message(payloadData1); - + var msg1 = new Message(payloadData1); + string payload2 = "kafka 2."; byte[] payloadData2 = Encoding.UTF8.GetBytes(payload2); - Message msg2 = new Message(payloadData2); + var msg2 = new Message(payloadData2); - var config = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(config); - var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List() { msg1, msg2 }); - producer.Send(producerRequest); + using (var producer = new SyncProducer(prodConfig)) + { + var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { msg1, msg2 }); + producer.Send(producerRequest); + } } /// @@ -78,12 +69,15 @@ [Test] public void ProducerSendsMessageWithLongTopic() { - Message msg = new Message(Encoding.UTF8.GetBytes("test message")); + var prodConfig = this.SyncProducerConfig1; + + var msg = new Message(Encoding.UTF8.GetBytes("test message")); string topic = "ThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopicThisIsAVeryLongTopic"; - var config = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(config); - var producerRequest = new ProducerRequest(topic, 0, new List() { msg }); - producer.Send(producerRequest); + using (var producer = new SyncProducer(prodConfig)) + { + var producerRequest = new ProducerRequest(topic, 0, new List { msg }); + producer.Send(producerRequest); + } } /// @@ -92,12 +86,12 @@ [Test] public void AsyncProducerSendsManyLongRandomMessages() { + var prodConfig = this.AsyncProducerConfig1; List messages = GenerateRandomTextMessages(50); - - var config = new AsyncProducerConfig(clientConfig); - - var producer = new AsyncProducer(config); - producer.Send(CurrentTestTopic, 0, messages); + using (var producer = new AsyncProducer(prodConfig)) + { + producer.Send(CurrentTestTopic, 0, messages); + } } /// @@ -106,7 +100,9 @@ [Test] public void AsyncProducerSendsFewShortFixedMessages() { - List messages = new List() + var prodConfig = this.AsyncProducerConfig1; + + var messages = new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")), new Message(Encoding.UTF8.GetBytes("Async Test Message 2")), @@ -114,10 +110,10 @@ new Message(Encoding.UTF8.GetBytes("Async Test Message 4")) }; - var config = new AsyncProducerConfig(clientConfig); - - var producer = new AsyncProducer(config); - producer.Send(CurrentTestTopic, 0, messages); + using (var producer = new AsyncProducer(prodConfig)) + { + producer.Send(CurrentTestTopic, 0, messages); + } } /// @@ -126,25 +122,26 @@ [Test] public void AsyncProducerSendsFewShortFixedMessagesInSeparateSendActions() { - var config = new AsyncProducerConfig(clientConfig); - using (var producer = new AsyncProducer(config)) + var prodConfig = this.AsyncProducerConfig1; + + using (var producer = new AsyncProducer(prodConfig)) { - ProducerRequest req1 = new ProducerRequest( + var req1 = new ProducerRequest( CurrentTestTopic, 0, - new List() { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")) }); + new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")) }); producer.Send(req1); - ProducerRequest req2 = new ProducerRequest( + var req2 = new ProducerRequest( CurrentTestTopic, 0, - new List() { new Message(Encoding.UTF8.GetBytes("Async Test Message 2")) }); + new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 2")) }); producer.Send(req2); - ProducerRequest req3 = new ProducerRequest( + var req3 = new ProducerRequest( CurrentTestTopic, 0, - new List() { new Message(Encoding.UTF8.GetBytes("Async Test Message 3")) }); + new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 3")) }); producer.Send(req3); } } @@ -152,14 +149,18 @@ [Test] public void AsyncProducerSendsMessageWithCallbackClass() { - List messages = new List() + var prodConfig = this.AsyncProducerConfig1; + + var messages = new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")), }; - var config = new AsyncProducerConfig(clientConfig); - TestCallbackHandler myHandler = new TestCallbackHandler(); - var producer = new AsyncProducer(config, myHandler); - producer.Send(CurrentTestTopic, 0, messages); + var myHandler = new TestCallbackHandler(); + using (var producer = new AsyncProducer(prodConfig, myHandler)) + { + producer.Send(CurrentTestTopic, 0, messages); + } + Thread.Sleep(1000); Assert.IsTrue(myHandler.WasRun); } @@ -167,14 +168,18 @@ [Test] public void AsyncProducerSendsMessageWithCallback() { - List messages = new List() + var prodConfig = this.AsyncProducerConfig1; + + var messages = new List { new Message(Encoding.UTF8.GetBytes("Async Test Message 1")), }; - var config = new AsyncProducerConfig(clientConfig); - TestCallbackHandler myHandler = new TestCallbackHandler(); - var producer = new AsyncProducer(config); - producer.Send(CurrentTestTopic, 0, messages, myHandler.Handle); + var myHandler = new TestCallbackHandler(); + using (var producer = new AsyncProducer(prodConfig)) + { + producer.Send(CurrentTestTopic, 0, messages, myHandler.Handle); + } + Thread.Sleep(1000); Assert.IsTrue(myHandler.WasRun); } @@ -195,7 +200,9 @@ [Test] public void ProducerSendMultiRequest() { - List requests = new List + var prodConfig = this.SyncProducerConfig1; + + var requests = new List { new ProducerRequest(CurrentTestTopic, 0, new List { new Message(Encoding.UTF8.GetBytes("1: " + DateTime.UtcNow)) }), new ProducerRequest(CurrentTestTopic, 0, new List { new Message(Encoding.UTF8.GetBytes("2: " + DateTime.UtcNow)) }), @@ -203,9 +210,10 @@ new ProducerRequest(CurrentTestTopic, 0, new List { new Message(Encoding.UTF8.GetBytes("4: " + DateTime.UtcNow)) }) }; - var config = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(config); - producer.MultiSend(requests); + using (var producer = new SyncProducer(prodConfig)) + { + producer.MultiSend(requests); + } } /// @@ -214,17 +222,21 @@ [Test] public void ConsumerFetchMessage() { + var consumerConfig = this.ConsumerConfig1; ProducerSendsMessage(); - - ConsumerConfig config = new ConsumerConfig(clientConfig); - IConsumer consumer = new Kafka.Client.Consumers.Consumer(config); - FetchRequest request = new FetchRequest(CurrentTestTopic, 0, 0); + Thread.Sleep(1000); + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, 0, 0); BufferedMessageSet response = consumer.Fetch(request); Assert.NotNull(response); - foreach (var message in response.Messages) + int count = 0; + foreach (var message in response) { - Console.WriteLine(message); + count++; + Console.WriteLine(message.Message); } + + Assert.AreEqual(2, count); } /// @@ -233,25 +245,28 @@ [Test] public void ConsumerMultiFetchGetsMessage() { - ProducerSendMultiRequest(); + var config = this.ConsumerConfig1; - ConsumerConfig config = new ConsumerConfig(clientConfig); - IConsumer cons = new Consumers.Consumer(config); - MultiFetchRequest request = new MultiFetchRequest(new List + ProducerSendMultiRequest(); + Thread.Sleep(2000); + IConsumer cons = new Consumer(config); + var request = new MultiFetchRequest(new List { new FetchRequest(CurrentTestTopic, 0, 0), new FetchRequest(CurrentTestTopic, 0, 0), - new FetchRequest(CurrentTestTopic + "2", 0, 0) + new FetchRequest(CurrentTestTopic, 0, 0) }); IList response = cons.MultiFetch(request); + Assert.AreEqual(3, response.Count); for (int ix = 0; ix < response.Count; ix++) { IEnumerable messageSet = response[ix].Messages; + Assert.AreEqual(4, messageSet.Count()); Console.WriteLine(string.Format("Request #{0}-->", ix)); foreach (Message msg in messageSet) { - Console.WriteLine(msg); + Console.WriteLine(msg.ToString()); } } } @@ -262,10 +277,10 @@ [Test] public void ConsumerGetsOffsets() { - OffsetRequest request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10); + var consumerConfig = this.ConsumerConfig1; - ConsumerConfig config = new ConsumerConfig(clientConfig); - IConsumer consumer = new Consumers.Consumer(config); + var request = new OffsetRequest(CurrentTestTopic, 0, DateTime.Now.AddHours(-24).Ticks, 10); + IConsumer consumer = new Consumer(consumerConfig); IList list = consumer.GetOffsetsBefore(request); foreach (long l in list) @@ -280,20 +295,19 @@ [Test] public void ProducerSendsAndConsumerReceivesSingleSimpleMessage() { - Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ConsumerConfig1; - var config = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(config); - var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List() { sourceMessage }); + var sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); + long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, consumerConfig); + using (var producer = new SyncProducer(prodConfig)) + { + var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { sourceMessage }); + producer.Send(producerRequest); + } - long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); - - producer.Send(producerRequest); - - ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); - IConsumer consumer = new Consumers.Consumer(consumerConfig); - FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); - + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, 0, currentOffset); BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; @@ -305,13 +319,11 @@ { break; } - else + + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) - { - break; - } + break; } } @@ -327,20 +339,20 @@ [Test] public void AsyncProducerSendsAndConsumerReceivesSingleSimpleMessage() { - Message sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); + var prodConfig = this.AsyncProducerConfig1; + var consumerConfig = this.ConsumerConfig1; - var config = new AsyncProducerConfig(clientConfig); - var producer = new AsyncProducer(config); - var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List() { sourceMessage }); + var sourceMessage = new Message(Encoding.UTF8.GetBytes("test message")); + using (var producer = new AsyncProducer(prodConfig)) + { + var producerRequest = new ProducerRequest(CurrentTestTopic, 0, new List { sourceMessage }); + producer.Send(producerRequest); + } - long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, clientConfig); + long currentOffset = TestHelper.GetCurrentKafkaOffset(CurrentTestTopic, consumerConfig); + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, 0, currentOffset); - producer.Send(producerRequest); - - ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); - IConsumer consumer = new Consumers.Consumer(consumerConfig); - FetchRequest request = new FetchRequest(CurrentTestTopic, 0, currentOffset); - BufferedMessageSet response; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; @@ -352,13 +364,11 @@ { break; } - else + + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) - { - break; - } + break; } } @@ -374,16 +384,19 @@ [Test] public void ProducerSendsAndConsumerReceivesMultiRequest() { + var prodConfig = this.SyncProducerConfig1; + var consumerConfig = this.ConsumerConfig1; + string testTopic1 = CurrentTestTopic + "1"; string testTopic2 = CurrentTestTopic + "2"; string testTopic3 = CurrentTestTopic + "3"; - Message sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage")); - Message sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage")); - Message sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage")); - Message sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage")); + var sourceMessage1 = new Message(Encoding.UTF8.GetBytes("1: TestMessage")); + var sourceMessage2 = new Message(Encoding.UTF8.GetBytes("2: TestMessage")); + var sourceMessage3 = new Message(Encoding.UTF8.GetBytes("3: TestMessage")); + var sourceMessage4 = new Message(Encoding.UTF8.GetBytes("4: TestMessage")); - List requests = new List + var requests = new List { new ProducerRequest(testTopic1, 0, new List { sourceMessage1 }), new ProducerRequest(testTopic1, 0, new List { sourceMessage2 }), @@ -391,18 +404,17 @@ new ProducerRequest(testTopic3, 0, new List { sourceMessage4 }) }; - var config = new SyncProducerConfig(clientConfig); - var producer = new SyncProducer(config); + long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, consumerConfig); + long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, consumerConfig); + long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, consumerConfig); - long currentOffset1 = TestHelper.GetCurrentKafkaOffset(testTopic1, clientConfig); - long currentOffset2 = TestHelper.GetCurrentKafkaOffset(testTopic2, clientConfig); - long currentOffset3 = TestHelper.GetCurrentKafkaOffset(testTopic3, clientConfig); + using (var producer = new SyncProducer(prodConfig)) + { + producer.MultiSend(requests); + } - producer.MultiSend(requests); - - ConsumerConfig consumerConfig = new ConsumerConfig(clientConfig); - IConsumer consumer = new Consumers.Consumer(consumerConfig); - MultiFetchRequest request = new MultiFetchRequest(new List + IConsumer consumer = new Consumer(consumerConfig); + var request = new MultiFetchRequest(new List { new FetchRequest(testTopic1, 0, currentOffset1), new FetchRequest(testTopic2, 0, currentOffset2), @@ -419,13 +431,11 @@ { break; } - else + + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) - { - break; - } + break; } } @@ -440,43 +450,13 @@ } /// - /// Gererates a randome list of messages. - /// - /// The number of messages to generate. - /// A list of random messages. - private static List GenerateRandomMessages(int numberOfMessages) - { - List messages = new List(); - for (int ix = 0; ix < numberOfMessages; ix++) - { - messages.Add(new Message(GenerateRandomBytes(10000))); - } - - return messages; - } - - /// - /// Generate a random set of bytes. - /// - /// Length of the byte array. - /// Random byte array. - private static byte[] GenerateRandomBytes(int length) - { - byte[] randBytes = new byte[length]; - Random randNum = new Random(); - randNum.NextBytes(randBytes); - - return randBytes; - } - - /// /// Gererates a randome list of text messages. /// /// The number of messages to generate. /// A list of random text messages. private static List GenerateRandomTextMessages(int numberOfMessages) { - List messages = new List(); + var messages = new List(); for (int ix = 0; ix < numberOfMessages; ix++) { ////messages.Add(new Message(GenerateRandomBytes(10000))); @@ -493,12 +473,11 @@ /// Random message string. private static string GenerateRandomMessage(int length) { - StringBuilder builder = new StringBuilder(); - Random random = new Random(); - char ch; + var builder = new StringBuilder(); + var random = new Random(); for (int i = 0; i < length; i++) { - ch = Convert.ToChar(Convert.ToInt32( + char ch = Convert.ToChar(Convert.ToInt32( Math.Floor((26 * random.NextDouble()) + 65))); builder.Append(ch); } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZKBrokerPartitionInfoTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZKBrokerPartitionInfoTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZKBrokerPartitionInfoTests.cs (working copy) @@ -19,7 +19,6 @@ { using System.Collections.Generic; using System.Linq; - using Kafka.Client.Cfg; using Kafka.Client.Cluster; using Kafka.Client.Producers.Partitioning; using Kafka.Client.Utils; @@ -29,60 +28,53 @@ using ZooKeeperNet; [TestFixture] - public class ZKBrokerPartitionInfoTests : IntegrationFixtureBase + public class ZkBrokerPartitionInfoTests : IntegrationFixtureBase { - private KafkaClientConfiguration clientConfig; - private ZKConfig zkConfig; - - [TestFixtureSetUp] - public void SetUp() + [Test] + public void ZkBrokerPartitionInfoGetsAllBrokerInfo() { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - zkConfig = new ProducerConfig(clientConfig); - } + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + var prodConfigNotZk = this.ConfigBasedSyncProdConfig; - [Test] - public void ZKBrokerPartitionInfoGetsAllBrokerInfo() - { IDictionary allBrokerInfo; - using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(zkConfig, null)) + using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(prodConfig, null)) { allBrokerInfo = brokerPartitionInfo.GetAllBrokerInfo(); } - Assert.AreEqual(clientConfig.BrokerPartitionInfos.Count, allBrokerInfo.Count); - foreach (BrokerPartitionInfo cfgBrPartInfo in clientConfig.BrokerPartitionInfos) - { - Assert.IsTrue(allBrokerInfo.ContainsKey(cfgBrPartInfo.Id)); - Assert.AreEqual(cfgBrPartInfo.Address, allBrokerInfo[cfgBrPartInfo.Id].Host); - Assert.AreEqual(cfgBrPartInfo.Port, allBrokerInfo[cfgBrPartInfo.Id].Port); - } + Assert.AreEqual(prodConfigNotZk.Brokers.Count, allBrokerInfo.Count); + allBrokerInfo.Values.All(x => prodConfigNotZk.Brokers.Any( + y => x.Id == y.BrokerId + && x.Host == y.Host + && x.Port == y.Port)); } [Test] - public void ZKBrokerPartitionInfoGetsBrokerPartitionInfo() + public void ZkBrokerPartitionInfoGetsBrokerPartitionInfo() { + var prodconfig = this.ZooKeeperBasedSyncProdConfig; SortedSet partitions; - using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(zkConfig, null)) + using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(prodconfig, null)) { partitions = brokerPartitionInfo.GetBrokerPartitionInfo("test"); } Assert.NotNull(partitions); Assert.GreaterOrEqual(partitions.Count, 2); - var partition = partitions.ToList()[0]; - Assert.AreEqual(0, partition.BrokerId); } [Test] public void ZkBrokerPartitionInfoGetsBrokerInfo() { - using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(zkConfig, null)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + var prodConfigNotZk = this.ConfigBasedSyncProdConfig; + + using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(prodConfig, null)) { - var testBroker = clientConfig.BrokerPartitionInfos[0]; - Broker broker = brokerPartitionInfo.GetBrokerInfo(testBroker.Id); + var testBroker = prodConfigNotZk.Brokers[0]; + Broker broker = brokerPartitionInfo.GetBrokerInfo(testBroker.BrokerId); Assert.NotNull(broker); - Assert.AreEqual(testBroker.Address, broker.Host); + Assert.AreEqual(testBroker.Host, broker.Host); Assert.AreEqual(testBroker.Port, broker.Port); } } @@ -90,14 +82,15 @@ [Test] public void WhenNewTopicIsAddedBrokerTopicsListenerCreatesNewMapping() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; IDictionary brokers; string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -114,8 +107,8 @@ Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); @@ -125,6 +118,7 @@ client.CreatePersistent(topicPath, true); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(topicPath); } @@ -134,13 +128,14 @@ [Test] public void WhenNewBrokerIsAddedBrokerTopicsListenerUpdatesBrokersList() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -157,18 +152,20 @@ Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); WaitUntillIdle(client, 500); var brokerTopicsListener = new BrokerTopicsListener(client, mappings, brokers, null); client.Subscribe(ZooKeeperClient.DefaultBrokerIdsPath, brokerTopicsListener); + WaitUntillIdle(client, 500); client.CreatePersistent(brokerPath, true); client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(brokerPath); } @@ -181,13 +178,14 @@ [Test] public void WhenBrokerIsRemovedBrokerTopicsListenerUpdatesBrokersList() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -204,8 +202,8 @@ Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); @@ -225,15 +223,16 @@ [Test] public void WhenNewBrokerInTopicIsAddedBrokerTopicsListenerUpdatesMappings() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic; string topicBrokerPath = topicPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -250,8 +249,8 @@ Assert.NotNull(mappings); Assert.Greater(mappings.Count, 0); using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); @@ -269,6 +268,7 @@ client.WriteData(topicBrokerPath, 5); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(brokerPath); client.DeleteRecursive(topicPath); } @@ -281,55 +281,56 @@ [Test] public void WhenSessionIsExpiredListenerRecreatesEphemeralNodes() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + IDictionary> mappings; + IDictionary brokers; + IDictionary> mappings2; + IDictionary brokers2; + using ( + IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { - var producerConfig = new ProducerConfig(clientConfig); - IDictionary> mappings; - IDictionary brokers; - IDictionary> mappings2; - IDictionary brokers2; - using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, - ZooKeeperStringSerializer.Serializer)) + using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) { - using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) - { - brokers = brokerPartitionInfo.GetAllBrokerInfo(); - mappings = - ReflectionHelper.GetInstanceField>>( - "topicBrokerPartitions", brokerPartitionInfo); - Assert.NotNull(brokers); - Assert.Greater(brokers.Count, 0); - Assert.NotNull(mappings); - Assert.Greater(mappings.Count, 0); - client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null)); - WaitUntillIdle(client, 3000); - brokers2 = brokerPartitionInfo.GetAllBrokerInfo(); - mappings2 = - ReflectionHelper.GetInstanceField>>( - "topicBrokerPartitions", brokerPartitionInfo); - } + brokers = brokerPartitionInfo.GetAllBrokerInfo(); + mappings = + ReflectionHelper.GetInstanceField>>( + "topicBrokerPartitions", brokerPartitionInfo); + Assert.NotNull(brokers); + Assert.Greater(brokers.Count, 0); + Assert.NotNull(mappings); + Assert.Greater(mappings.Count, 0); + client.Process(new WatchedEvent(KeeperState.Expired, EventType.None, null)); + WaitUntillIdle(client, 3000); + brokers2 = brokerPartitionInfo.GetAllBrokerInfo(); + mappings2 = + ReflectionHelper.GetInstanceField>>( + "topicBrokerPartitions", brokerPartitionInfo); } + } - Assert.NotNull(brokers2); - Assert.Greater(brokers2.Count, 0); - Assert.NotNull(mappings2); - Assert.Greater(mappings2.Count, 0); - Assert.AreEqual(brokers.Count, brokers2.Count); - Assert.AreEqual(mappings.Count, mappings2.Count); - } + Assert.NotNull(brokers2); + Assert.Greater(brokers2.Count, 0); + Assert.NotNull(mappings2); + Assert.Greater(mappings2.Count, 0); + Assert.AreEqual(brokers.Count, brokers2.Count); + Assert.AreEqual(mappings.Count, mappings2.Count); } [Test] - public void WhenNewTopicIsAddedZKBrokerPartitionInfoUpdatesMappings() + public void WhenNewTopicIsAddedZkBrokerPartitionInfoUpdatesMappings() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -340,6 +341,7 @@ client.CreatePersistent(topicPath, true); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(topicPath); } } @@ -350,14 +352,15 @@ } [Test] - public void WhenNewBrokerIsAddedZKBrokerPartitionInfoUpdatesBrokersList() + public void WhenNewBrokerIsAddedZkBrokerPartitionInfoUpdatesBrokersList() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -367,6 +370,7 @@ client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(brokerPath); } } @@ -380,18 +384,20 @@ } [Test] - public void WhenBrokerIsRemovedZKBrokerPartitionInfoUpdatesBrokersList() + public void WhenBrokerIsRemovedZkBrokerPartitionInfoUpdatesBrokersList() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) { + WaitUntillIdle(client, 500); brokers = brokerPartitionInfo.GetAllBrokerInfo(); client.CreatePersistent(brokerPath, true); client.WriteData(brokerPath, "192.168.1.39-1310449279123:192.168.1.39:9102"); @@ -410,17 +416,18 @@ } [Test] - public void WhenNewBrokerInTopicIsAddedZKBrokerPartitionInfoUpdatesMappings() + public void WhenNewBrokerInTopicIsAddedZkBrokerPartitionInfoUpdatesMappings() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IDictionary> mappings; IDictionary brokers; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string topicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/" + CurrentTestTopic; string topicBrokerPath = topicPath + "/" + 2345; using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { using (var brokerPartitionInfo = new ZKBrokerPartitionInfo(client)) @@ -439,6 +446,7 @@ client.WriteData(topicBrokerPath, 5); WaitUntillIdle(client, 500); client.UnsubscribeAll(); + WaitUntillIdle(client, 500); client.DeleteRecursive(brokerPath); client.DeleteRecursive(topicPath); } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestMultipleBrokersHelper.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestMultipleBrokersHelper.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/TestMultipleBrokersHelper.cs (working copy) @@ -22,57 +22,54 @@ public class TestMultipleBrokersHelper { - private BrokerPartitionInfoCollection configBrokers = - KafkaClientConfiguration.GetConfiguration().BrokerPartitionInfos; + private readonly Dictionary> offsets = new Dictionary>(); - private Dictionary offsets = new Dictionary(); + private readonly string topic; - private BrokerPartitionInfo changedBroker; - - private string topic; - public TestMultipleBrokersHelper(string topic) { this.topic = topic; } - public BrokerPartitionInfo BrokerThatHasChanged - { - get { return changedBroker; } - } + public SyncProducerConfiguration BrokerThatHasChanged { get; private set; } + public int PartitionThatHasChanged { get; private set; } + public long OffsetFromBeforeTheChange { get { - if (changedBroker != null) - { - return offsets[changedBroker.Id]; - } - else - { - return 0; - } + return this.BrokerThatHasChanged != null ? this.offsets[this.BrokerThatHasChanged.BrokerId][this.PartitionThatHasChanged] : 0; } } - public void GetCurrentOffsets() + public void GetCurrentOffsets(IEnumerable brokers) { - foreach (BrokerPartitionInfo broker in configBrokers) + foreach (var broker in brokers) { - offsets.Add(broker.Id, TestHelper.GetCurrentKafkaOffset(topic, broker.Address, broker.Port)); + offsets.Add(broker.BrokerId, new Dictionary()); + offsets[broker.BrokerId].Add(0, TestHelper.GetCurrentKafkaOffset(topic, broker.Host, broker.Port, 0)); + offsets[broker.BrokerId].Add(1, TestHelper.GetCurrentKafkaOffset(topic, broker.Host, broker.Port, 1)); } } - public bool CheckIfAnyBrokerHasChanged() + public bool CheckIfAnyBrokerHasChanged(IEnumerable brokers) { - foreach (BrokerPartitionInfo broker in configBrokers) + foreach (var broker in brokers) { - if (TestHelper.GetCurrentKafkaOffset(topic, broker.Address, broker.Port) != offsets[broker.Id]) + if (TestHelper.GetCurrentKafkaOffset(topic, broker.Host, broker.Port, 0) != offsets[broker.BrokerId][0]) { - changedBroker = broker; + this.BrokerThatHasChanged = broker; + this.PartitionThatHasChanged = 0; return true; } + + if (TestHelper.GetCurrentKafkaOffset(topic, broker.Host, broker.Port, 1) != offsets[broker.BrokerId][1]) + { + this.BrokerThatHasChanged = broker; + this.PartitionThatHasChanged = 1; + return true; + } } return false; Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/IntegrationFixtureBase.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/IntegrationFixtureBase.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/IntegrationFixtureBase.cs (working copy) @@ -19,6 +19,7 @@ { using System; using System.Threading; + using Kafka.Client.Cfg; using Kafka.Client.ZooKeeperIntegration; using NUnit.Framework; @@ -26,21 +27,121 @@ { protected string CurrentTestTopic { get; set; } + protected ProducerConfiguration ConfigBasedSyncProdConfig + { + get + { + return ProducerConfiguration.Configure(ProducerConfiguration.DefaultSectionName); + } + } + + protected SyncProducerConfiguration SyncProducerConfig1 + { + get + { + var prodConfig = this.ConfigBasedSyncProdConfig; + return new SyncProducerConfiguration( + prodConfig, + prodConfig.Brokers[0].BrokerId, + prodConfig.Brokers[0].Host, + prodConfig.Brokers[0].Port); + } + } + + protected SyncProducerConfiguration SyncProducerConfig2 + { + get + { + var prodConfig = this.ConfigBasedSyncProdConfig; + return new SyncProducerConfiguration( + prodConfig, + prodConfig.Brokers[1].BrokerId, + prodConfig.Brokers[1].Host, + prodConfig.Brokers[1].Port); + } + } + + protected SyncProducerConfiguration SyncProducerConfig3 + { + get + { + var prodConfig = this.ConfigBasedSyncProdConfig; + return new SyncProducerConfiguration( + prodConfig, + prodConfig.Brokers[2].BrokerId, + prodConfig.Brokers[2].Host, + prodConfig.Brokers[2].Port); + } + } + + protected ProducerConfiguration ZooKeeperBasedSyncProdConfig + { + get + { + return ProducerConfiguration.Configure(ProducerConfiguration.DefaultSectionName + 2); + } + } + + protected AsyncProducerConfiguration AsyncProducerConfig1 + { + get + { + var asyncUberConfig = ProducerConfiguration.Configure(ProducerConfiguration.DefaultSectionName + 3); + return new AsyncProducerConfiguration( + asyncUberConfig, + asyncUberConfig.Brokers[0].BrokerId, + asyncUberConfig.Brokers[0].Host, + asyncUberConfig.Brokers[0].Port); + } + } + + protected ConsumerConfiguration ConsumerConfig1 + { + get + { + return ConsumerConfiguration.Configure(ConsumerConfiguration.DefaultSection + 1); + } + } + + protected ConsumerConfiguration ConsumerConfig2 + { + get + { + return ConsumerConfiguration.Configure(ConsumerConfiguration.DefaultSection + 2); + } + } + + protected ConsumerConfiguration ConsumerConfig3 + { + get + { + return ConsumerConfiguration.Configure(ConsumerConfiguration.DefaultSection + 3); + } + } + + protected ConsumerConfiguration ZooKeeperBasedConsumerConfig + { + get + { + return ConsumerConfiguration.Configure(ConsumerConfiguration.DefaultSection + 4); + } + } + [SetUp] public void SetupCurrentTestTopic() { - CurrentTestTopic = TestContext.CurrentContext.Test.Name + "_" + Guid.NewGuid().ToString(); + CurrentTestTopic = TestContext.CurrentContext.Test.Name + "_" + Guid.NewGuid(); } internal static void WaitUntillIdle(IZooKeeperClient client, int timeout) { Thread.Sleep(timeout); - int rest = timeout - client.IdleTime; + int rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout; while (rest > 0) { Thread.Sleep(rest); - rest = timeout - client.IdleTime; + rest = client.IdleTime.HasValue ? timeout - client.IdleTime.Value : timeout; } } } -} \ No newline at end of file +} Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperClientTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperClientTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperClientTests.cs (working copy) @@ -35,16 +35,8 @@ internal class ZooKeeperClientTests : IntegrationFixtureBase, IZooKeeperDataListener, IZooKeeperStateListener, IZooKeeperChildListener { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - - private KafkaClientConfiguration clientConfig; private readonly IList events = new List(); - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - [SetUp] public void TestSetup() { @@ -54,8 +46,12 @@ [Test] public void ZooKeeperClientCreateWorkerThreadsOnBeingCreated() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); var eventWorker = ReflectionHelper.GetInstanceField("eventWorker", client); @@ -68,8 +64,12 @@ [Test] public void ZooKeeperClientFailsWhenCreatedWithWrongConnectionInfo() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient("random text", producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + "random text", + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { Assert.Throws(client.Connect); } @@ -78,8 +78,12 @@ [Test] public void WhenStateChangedToConnectedStateListenerFires() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Subscribe(this); client.Connect(); @@ -96,8 +100,12 @@ [Test] public void WhenStateChangedToDisconnectedStateListenerFires() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Subscribe(this); client.Connect(); @@ -116,8 +124,12 @@ [Test] public void WhenStateChangedToExpiredStateAndSessionListenersFire() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Subscribe(this); client.Connect(); @@ -143,10 +155,14 @@ [Test] public void WhenSessionExpiredClientReconnects() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IZooKeeperConnection conn1; IZooKeeperConnection conn2; - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); conn1 = ReflectionHelper.GetInstanceField("connection", client); @@ -161,8 +177,12 @@ [Test] public void ZooKeeperClientChecksIfPathExists() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); Assert.IsTrue(client.Exists(ZooKeeperClient.DefaultBrokerTopicsPath, false)); @@ -172,8 +192,12 @@ [Test] public void ZooKeeperClientCreatesANewPathAndDeletesIt() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); string myPath = "/" + Guid.NewGuid(); @@ -187,9 +211,13 @@ [Test] public void WhenChildIsCreatedChilListenerOnParentFires() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + string myPath = "/" + Guid.NewGuid(); - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); WaitUntillIdle(client, 500); @@ -212,9 +240,13 @@ [Test] public void WhenChildIsDeletedChildListenerOnParentFires() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + string myPath = "/" + Guid.NewGuid(); - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); @@ -236,9 +268,13 @@ [Test] public void WhenZNodeIsDeletedChildAndDataDeletedListenersFire() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + string myPath = "/" + Guid.NewGuid(); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); @@ -265,8 +301,12 @@ [Test] public void ZooKeeperClientCreatesAChildAndGetsChildren() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); string child = Guid.NewGuid().ToString(); @@ -284,11 +324,15 @@ [Test] public void WhenDataChangedDataListenerFires() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + string myPath = "/" + Guid.NewGuid(); string sourceData = "my test data"; string resultData; - using (IZooKeeperClient client = new ZooKeeperClient(producerConfig.ZkConnect, producerConfig.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (IZooKeeperClient client = new ZooKeeperClient( + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, + ZooKeeperStringSerializer.Serializer)) { client.Connect(); client.CreatePersistent(myPath, true); @@ -317,11 +361,12 @@ [ExpectedException(typeof(ZooKeeperException))] public void WhenClientWillNotConnectWithinGivenTimeThrows() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + using (IZooKeeperClient client = new ZooKeeperClient( - producerConfig.ZkConnect, - producerConfig.ZkSessionTimeoutMs, + prodConfig.ZooKeeper.ZkConnect, + prodConfig.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer, 1)) { Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Kafka.Client.IntegrationTests.csproj =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Kafka.Client.IntegrationTests.csproj (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Kafka.Client.IntegrationTests.csproj (working copy) @@ -88,6 +88,7 @@ + @@ -110,6 +111,9 @@ + + Settings.StyleCop + Designer @@ -134,4 +138,4 @@ - \ No newline at end of file + Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/MockAlwaysZeroPartitioner.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/MockAlwaysZeroPartitioner.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/MockAlwaysZeroPartitioner.cs (working copy) @@ -19,6 +19,8 @@ namespace Kafka.Client.IntegrationTests { + using Kafka.Client.Producers.Partitioning; + /// /// This mock partitioner will always point to the first partition (the one of index = 0) /// Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Config/Integration/App.config =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Config/Integration/App.config (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/Config/Integration/App.config (working copy) @@ -26,7 +26,7 @@ - + @@ -34,4 +34,4 @@ - \ No newline at end of file + Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerRebalancingTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerRebalancingTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ConsumerRebalancingTests.cs (working copy) @@ -19,8 +19,6 @@ { using System.Collections.Generic; using System.Linq; - using System.Threading; - using Kafka.Client.Cfg; using Kafka.Client.Cluster; using Kafka.Client.Consumers; using Kafka.Client.Utils; @@ -30,24 +28,13 @@ [TestFixture] public class ConsumerRebalancingTests : IntegrationFixtureBase { - /// - /// Kafka Client configuration - /// - private static KafkaClientConfiguration clientConfig; - - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - [Test] public void ConsumerPorformsRebalancingOnStart() { - var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1" }; + var config = this.ZooKeeperBasedConsumerConfig; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) { - ZooKeeperClient client = ReflectionHelper.GetInstanceField("zkClient", consumerConnector); + var client = ReflectionHelper.GetInstanceField("zkClient", consumerConnector); Assert.IsNotNull(client); client.DeleteRecursive("/consumers/group1"); var topicCount = new Dictionary { { "test", 1 } }; @@ -71,7 +58,7 @@ Assert.That(children, Is.Not.Null.And.Not.Empty); Assert.That(children.Count, Is.EqualTo(2)); string partId = children[0]; - string data = client.ReadData("/consumers/group1/owners/test/" + partId); + var data = client.ReadData("/consumers/group1/owners/test/" + partId); Assert.That(data, Is.Not.Null.And.Not.Empty); Assert.That(data, Contains.Substring(consumerId)); data = client.ReadData("/consumers/group1/ids/" + consumerId); @@ -79,7 +66,7 @@ Assert.That(data, Is.EqualTo("{ \"test\": 1 }")); } - using (var client = new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + using (var client = new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { client.Connect(); //// Should be created as ephemeral @@ -94,13 +81,7 @@ [Test] public void ConsumerPorformsRebalancingWhenNewBrokerIsAddedToTopic() { - var config = new ConsumerConfig(clientConfig) - { - AutoCommit = false, - GroupId = "group1", - ZkSessionTimeoutMs = 60000, - ZkConnectionTimeoutMs = 60000 - }; + var config = this.ZooKeeperBasedConsumerConfig; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) @@ -120,7 +101,7 @@ children = client.GetChildren("/consumers/group1/owners/test", false); Assert.That(children.Count, Is.EqualTo(3)); Assert.That(children, Contains.Item("2345-0")); - string data = client.ReadData("/consumers/group1/owners/test/2345-0"); + var data = client.ReadData("/consumers/group1/owners/test/2345-0"); Assert.That(data, Is.Not.Null); Assert.That(data, Contains.Substring(consumerId)); var topicRegistry = @@ -138,7 +119,7 @@ [Test] public void ConsumerPorformsRebalancingWhenBrokerIsRemovedFromTopic() { - var config = new ConsumerConfig(clientConfig) { AutoCommit = false, GroupId = "group1", ZkSessionTimeoutMs = 60000, ZkConnectionTimeoutMs = 60000 }; + var config = this.ZooKeeperBasedConsumerConfig; string brokerPath = ZooKeeperClient.DefaultBrokerIdsPath + "/" + 2345; string brokerTopicPath = ZooKeeperClient.DefaultBrokerTopicsPath + "/test/" + 2345; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) @@ -170,14 +151,7 @@ [Test] public void ConsumerPerformsRebalancingWhenNewConsumerIsAddedAndTheyDividePartitions() { - var config = new ConsumerConfig(clientConfig) - { - AutoCommit = false, - GroupId = "group1", - ZkSessionTimeoutMs = 60000, - ZkConnectionTimeoutMs = 60000 - }; - + var config = this.ZooKeeperBasedConsumerConfig; IList ids; IList owners; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) @@ -216,14 +190,8 @@ [Test] public void ConsumerPerformsRebalancingWhenConsumerIsRemovedAndTakesItsPartitions() { - var config = new ConsumerConfig(clientConfig) - { - AutoCommit = false, - GroupId = "group1", - ZkSessionTimeoutMs = 60000, - ZkConnectionTimeoutMs = 60000 - }; - + var config = this.ZooKeeperBasedConsumerConfig; + string basePath = "/consumers/" + config.GroupId; IList ids; IList owners; using (var consumerConnector = new ZookeeperConsumerConnector(config, true)) Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ProducerTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ProducerTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ProducerTests.cs (working copy) @@ -33,213 +33,194 @@ public class ProducerTests : IntegrationFixtureBase { /// - /// Kafka Client configuration - /// - private KafkaClientConfiguration clientConfig; - - /// /// Maximum amount of time to wait trying to get a specific test message from Kafka server (in miliseconds) /// private readonly int maxTestWaitTimeInMiliseconds = 5000; - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - clientConfig.SupressZooKeeper(); - } - [Test] public void ProducerSends1Message() { + var prodConfig = this.ConfigBasedSyncProdConfig; + int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); - - var producerConfig = new ProducerConfig(clientConfig); - var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new DefaultEncoder(), null)) + multipleBrokersHelper.GetCurrentOffsets( + new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); + using (var producer = new Producer(prodConfig)) { var producerData = new ProducerData( - CurrentTestTopic, "somekey", new List { originalMessage }); + CurrentTestTopic, new List { originalMessage }); producer.Send(producerData); Thread.Sleep(waitSingle); + } - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + while ( + !multipleBrokersHelper.CheckIfAnyBrokerHasChanged( + new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) + { + totalWaitTimeInMiliseconds += waitSingle; + Thread.Sleep(waitSingle); + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) - { - Assert.Fail("None of the brokers changed their offset after sending a message"); - } + Assert.Fail("None of the brokers changed their offset after sending a message"); } + } - totalWaitTimeInMiliseconds = 0; + totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); - var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); - - BufferedMessageSet response; - - while (true) + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); + var request1 = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); + BufferedMessageSet response; + while (true) + { + Thread.Sleep(waitSingle); + response = consumer.Fetch(request1); + if (response != null && response.Messages.Count() > 0) { - Thread.Sleep(waitSingle); - response = consumer.Fetch(request); - if (response != null && response.Messages.Count() > 0) - { - break; - } - - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) - { - break; - } + break; } - Assert.NotNull(response); - Assert.AreEqual(1, response.Messages.Count()); - Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) + { + break; + } } + + Assert.NotNull(response); + Assert.AreEqual(1, response.Messages.Count()); + Assert.AreEqual(originalMessage.ToString(), response.Messages.First().ToString()); } [Test] public void ProducerSends3Messages() { + var prodConfig = this.ConfigBasedSyncProdConfig; + int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1")); var originalMessage2 = new Message(Encoding.UTF8.GetBytes("TestData2")); var originalMessage3 = new Message(Encoding.UTF8.GetBytes("TestData3")); - var originalMessageList = - new List { originalMessage1, originalMessage2, originalMessage3 }; + var originalMessageList = new List { originalMessage1, originalMessage2, originalMessage3 }; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); - - var producerConfig = new ProducerConfig(clientConfig); - var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new DefaultEncoder(), null)) + multipleBrokersHelper.GetCurrentOffsets( + new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); + using (var producer = new Producer(prodConfig)) { - var producerData = new ProducerData(CurrentTestTopic, "somekey", originalMessageList); + var producerData = new ProducerData(CurrentTestTopic, originalMessageList); producer.Send(producerData); - Thread.Sleep(waitSingle); + } - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + Thread.Sleep(waitSingle); + while ( + !multipleBrokersHelper.CheckIfAnyBrokerHasChanged( + new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) + { + totalWaitTimeInMiliseconds += waitSingle; + Thread.Sleep(waitSingle); + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) - { - Assert.Fail("None of the brokers changed their offset after sending a message"); - } + Assert.Fail("None of the brokers changed their offset after sending a message"); } + } - totalWaitTimeInMiliseconds = 0; + totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); - var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); - BufferedMessageSet response; - while (true) + BufferedMessageSet response; + while (true) + { + Thread.Sleep(waitSingle); + response = consumer.Fetch(request); + if (response != null && response.Messages.Count() > 2) { - Thread.Sleep(waitSingle); - response = consumer.Fetch(request); - if (response != null && response.Messages.Count() > 2) - { - break; - } - - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) - { - break; - } + break; } - Assert.NotNull(response); - Assert.AreEqual(3, response.Messages.Count()); - Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString()); - Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString()); - Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString()); + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) + { + break; + } } + + Assert.NotNull(response); + Assert.AreEqual(3, response.Messages.Count()); + Assert.AreEqual(originalMessage1.ToString(), response.Messages.First().ToString()); + Assert.AreEqual(originalMessage2.ToString(), response.Messages.Skip(1).First().ToString()); + Assert.AreEqual(originalMessage3.ToString(), response.Messages.Skip(2).First().ToString()); } [Test] public void ProducerSends1MessageUsingNotDefaultEncoder() { + var prodConfig = this.ConfigBasedSyncProdConfig; + int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; string originalMessage = "TestData"; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); - - var producerConfig = new ProducerConfig(clientConfig); - var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new StringEncoder(), null)) + multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); + using (var producer = new Producer(prodConfig, null, new StringEncoder(), null)) { var producerData = new ProducerData( - CurrentTestTopic, "somekey", new List { originalMessage }); + CurrentTestTopic, new List { originalMessage }); producer.Send(producerData); - Thread.Sleep(waitSingle); + } - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + Thread.Sleep(waitSingle); + + while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) + { + totalWaitTimeInMiliseconds += waitSingle; + Thread.Sleep(waitSingle); + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { - totalWaitTimeInMiliseconds += waitSingle; - Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) - { - Assert.Fail("None of the brokers changed their offset after sending a message"); - } + Assert.Fail("None of the brokers changed their offset after sending a message"); } + } - totalWaitTimeInMiliseconds = 0; + totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); - var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, + multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); - BufferedMessageSet response; - while (true) + BufferedMessageSet response; + while (true) + { + Thread.Sleep(waitSingle); + response = consumer.Fetch(request); + if (response != null && response.Messages.Count() > 0) { - Thread.Sleep(waitSingle); - response = consumer.Fetch(request); - if (response != null && response.Messages.Count() > 0) - { - break; - } - - totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) - { - break; - } + break; } - Assert.NotNull(response); - Assert.AreEqual(1, response.Messages.Count()); - Assert.AreEqual(originalMessage, Encoding.UTF8.GetString(response.Messages.First().Payload)); + totalWaitTimeInMiliseconds += waitSingle; + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) + { + break; + } } + + Assert.NotNull(response); + Assert.AreEqual(1, response.Messages.Count()); + Assert.AreEqual(originalMessage, Encoding.UTF8.GetString(response.Messages.First().Payload)); } } } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperConnectionTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperConnectionTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperConnectionTests.cs (working copy) @@ -25,21 +25,14 @@ using ZooKeeperNet; [TestFixture] - public class ZooKeeperConnectionTests + public class ZooKeeperConnectionTests : IntegrationFixtureBase { - private KafkaClientConfiguration clientConfig; - - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - [Test] public void ZooKeeperConnectionCreatesAndDeletesPath() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperConnection connection = new ZooKeeperConnection(producerConfig.ZkConnect)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperConnection connection = new ZooKeeperConnection(prodConfig.ZooKeeper.ZkConnect)) { connection.Connect(null); string pathName = "/" + Guid.NewGuid(); @@ -53,9 +46,10 @@ [Test] public void ZooKeeperConnectionConnectsAndDisposes() { - var producerConfig = new ProducerConfig(clientConfig); + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + IZooKeeperConnection connection; - using (connection = new ZooKeeperConnection(producerConfig.ZkConnect)) + using (connection = new ZooKeeperConnection(prodConfig.ZooKeeper.ZkConnect)) { Assert.IsNull(connection.ClientState); connection.Connect(null); @@ -69,8 +63,9 @@ [Test] public void ZooKeeperConnectionCreatesAndGetsCreateTime() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperConnection connection = new ZooKeeperConnection(producerConfig.ZkConnect)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperConnection connection = new ZooKeeperConnection(prodConfig.ZooKeeper.ZkConnect)) { connection.Connect(null); string pathName = "/" + Guid.NewGuid(); @@ -84,8 +79,9 @@ [Test] public void ZooKeeperConnectionCreatesAndGetsChildren() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperConnection connection = new ZooKeeperConnection(producerConfig.ZkConnect)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperConnection connection = new ZooKeeperConnection(prodConfig.ZooKeeper.ZkConnect)) { connection.Connect(null); string child = Guid.NewGuid().ToString(); @@ -101,14 +97,15 @@ [Test] public void ZooKeeperConnectionWritesAndReadsData() { - var producerConfig = new ProducerConfig(clientConfig); - using (IZooKeeperConnection connection = new ZooKeeperConnection(producerConfig.ZkConnect)) + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + + using (IZooKeeperConnection connection = new ZooKeeperConnection(prodConfig.ZooKeeper.ZkConnect)) { connection.Connect(null); string child = Guid.NewGuid().ToString(); string pathName = "/" + child; connection.Create(pathName, null, CreateMode.Persistent); - var sourceData = new byte[2] { 1, 2 }; + var sourceData = new byte[] { 1, 2 }; connection.WriteData(pathName, sourceData); byte[] resultData = connection.ReadData(pathName, null, false); Assert.IsNotNull(resultData); Index: clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperAwareProducerTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperAwareProducerTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.IntegrationTests/ZooKeeperAwareProducerTests.cs (working copy) @@ -33,44 +33,34 @@ public class ZooKeeperAwareProducerTests : IntegrationFixtureBase { /// - /// Kafka Client configuration - /// - private KafkaClientConfiguration clientConfig; - - /// /// Maximum amount of time to wait trying to get a specific test message from Kafka server (in miliseconds) /// - private readonly int MaxTestWaitTimeInMiliseconds = 5000; + private readonly int maxTestWaitTimeInMiliseconds = 5000; - [TestFixtureSetUp] - public void SetUp() - { - clientConfig = KafkaClientConfiguration.GetConfiguration(); - } - [Test] - public void ZKAwareProducerSends1Message() + public void ZkAwareProducerSends1Message() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage = new Message(Encoding.UTF8.GetBytes("TestData")); var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); + multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); - var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new DefaultEncoder())) + using (var producer = new Producer(prodConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData( - CurrentTestTopic, "somekey", new List() { originalMessage }); + CurrentTestTopic, "somekey", new List { originalMessage }); producer.Send(producerData); - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } @@ -78,13 +68,11 @@ totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); - var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, + multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); + var request = new FetchRequest(CurrentTestTopic, multipleBrokersHelper.PartitionThatHasChanged, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; @@ -98,7 +86,7 @@ } totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } @@ -113,6 +101,7 @@ [Test] public void ZkAwareProducerSends3Messages() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; var originalMessage1 = new Message(Encoding.UTF8.GetBytes("TestData1")); @@ -121,20 +110,19 @@ var originalMessageList = new List { originalMessage1, originalMessage2, originalMessage3 }; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); + multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); - var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new DefaultEncoder())) + using (var producer = new Producer(prodConfig, mockPartitioner, new DefaultEncoder())) { var producerData = new ProducerData(CurrentTestTopic, "somekey", originalMessageList); producer.Send(producerData); - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } @@ -142,12 +130,10 @@ totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, + multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; @@ -161,7 +147,7 @@ } totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } @@ -178,26 +164,27 @@ [Test] public void ZkAwareProducerSends1MessageUsingNotDefaultEncoder() { + var prodConfig = this.ZooKeeperBasedSyncProdConfig; + int totalWaitTimeInMiliseconds = 0; int waitSingle = 100; string originalMessage = "TestData"; var multipleBrokersHelper = new TestMultipleBrokersHelper(CurrentTestTopic); - multipleBrokersHelper.GetCurrentOffsets(); + multipleBrokersHelper.GetCurrentOffsets(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 }); - var producerConfig = new ProducerConfig(clientConfig); var mockPartitioner = new MockAlwaysZeroPartitioner(); - using (var producer = new Producer(producerConfig, mockPartitioner, new StringEncoder(), null)) + using (var producer = new Producer(prodConfig, mockPartitioner, new StringEncoder(), null)) { var producerData = new ProducerData( CurrentTestTopic, "somekey", new List { originalMessage }); producer.Send(producerData); - while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged()) + while (!multipleBrokersHelper.CheckIfAnyBrokerHasChanged(new[] { this.SyncProducerConfig1, this.SyncProducerConfig2, this.SyncProducerConfig3 })) { totalWaitTimeInMiliseconds += waitSingle; Thread.Sleep(waitSingle); - if (totalWaitTimeInMiliseconds > MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds > this.maxTestWaitTimeInMiliseconds) { Assert.Fail("None of the brokers changed their offset after sending a message"); } @@ -205,12 +192,10 @@ totalWaitTimeInMiliseconds = 0; - var consumerConfig = new ConsumerConfig(clientConfig) - { - Host = multipleBrokersHelper.BrokerThatHasChanged.Address, - Port = multipleBrokersHelper.BrokerThatHasChanged.Port - }; - IConsumer consumer = new Consumers.Consumer(consumerConfig); + var consumerConfig = new ConsumerConfiguration( + multipleBrokersHelper.BrokerThatHasChanged.Host, + multipleBrokersHelper.BrokerThatHasChanged.Port); + IConsumer consumer = new Consumer(consumerConfig); var request = new FetchRequest(CurrentTestTopic, 0, multipleBrokersHelper.OffsetFromBeforeTheChange); BufferedMessageSet response; @@ -224,7 +209,7 @@ } totalWaitTimeInMiliseconds += waitSingle; - if (totalWaitTimeInMiliseconds >= MaxTestWaitTimeInMiliseconds) + if (totalWaitTimeInMiliseconds >= this.maxTestWaitTimeInMiliseconds) { break; } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Producers/PartitioningTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Producers/PartitioningTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Producers/PartitioningTests.cs (working copy) @@ -21,17 +21,20 @@ using Kafka.Client.Cluster; using Kafka.Client.Producers.Partitioning; using NUnit.Framework; + using System.Collections.Generic; [TestFixture] public class PartitioningTests { - private ProducerConfig config; + private ProducerConfiguration config; [TestFixtureSetUp] public void SetUp() { - config = new ProducerConfig(); - config.BrokerPartitionInfo = "1:192.168.0.1:1234,2:192.168.0.2:3456"; + var brokers = new List(); + brokers.Add(new BrokerConfiguration { BrokerId = 1, Host = "192.168.0.1", Port = 1234 }); + brokers.Add(new BrokerConfiguration { BrokerId = 2, Host = "192.168.0.2", Port = 3456 }); + config = new ProducerConfiguration(brokers); } [Test] Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageSetTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageSetTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageSetTests.cs (working copy) @@ -29,12 +29,14 @@ { private const int MessageLengthPartLength = 4; private const int MagicNumberPartLength = 1; + private const int AttributesPartLength = 1; private const int ChecksumPartLength = 4; - + private const int MessageLengthPartOffset = 0; private const int MagicNumberPartOffset = 4; - private const int ChecksumPartOffset = 5; - private const int DataPartOffset = 9; + private const int AttributesPartOffset = 5; + private const int ChecksumPartOffset = 6; + private const int DataPartOffset = 10; [Test] public void BufferedMessageSetWriteToValidSequence() @@ -55,9 +57,9 @@ Array.Reverse(messageLength); } - Assert.AreEqual(MagicNumberPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); + Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); - Assert.AreEqual(0, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 0 + Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1 byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); @@ -68,7 +70,7 @@ Assert.AreEqual(messageBytes, dataPart); ////second message - int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + ChecksumPartLength + + int secondMessageOffset = MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length; messageLength = new byte[MessageLengthPartLength]; @@ -78,9 +80,9 @@ Array.Reverse(messageLength); } - Assert.AreEqual(MagicNumberPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); + Assert.AreEqual(MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length, BitConverter.ToInt32(messageLength, 0)); - Assert.AreEqual(0, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]); // default magic number should be 0 + Assert.AreEqual(1, ms.ToArray()[secondMessageOffset + MagicNumberPartOffset]); // default magic number should be 1 checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), secondMessageOffset + ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); @@ -99,7 +101,7 @@ Message msg2 = new Message(messageBytes); MessageSet messageSet = new BufferedMessageSet(new List() { msg1, msg2 }); Assert.AreEqual( - 2 * (MessageLengthPartLength + MagicNumberPartLength + ChecksumPartLength + messageBytes.Length), + 2 * (MessageLengthPartLength + MagicNumberPartLength + AttributesPartLength + ChecksumPartLength + messageBytes.Length), messageSet.SetSize); } } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/ProducerRequestTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/ProducerRequestTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/ProducerRequestTests.cs (working copy) @@ -50,10 +50,10 @@ byte[] bytes = ms.ToArray(); Assert.IsNotNull(bytes); - Assert.AreEqual(40, bytes.Length); + Assert.AreEqual(41, bytes.Length); // next 4 bytes = the length of the request - Assert.AreEqual(36, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray()), 0)); + Assert.AreEqual(37, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray()), 0)); // next 2 bytes = the RequestType which in this case should be Produce Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray()), 0)); @@ -68,10 +68,10 @@ Assert.AreEqual(0, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(13).Take(4).ToArray()), 0)); // next 4 bytes = the length of the individual messages in the pack - Assert.AreEqual(19, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray()), 0)); + Assert.AreEqual(20, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(17).Take(4).ToArray()), 0)); // fianl bytes = the individual messages in the pack - Assert.AreEqual(19, bytes.Skip(21).ToArray().Length); + Assert.AreEqual(20, bytes.Skip(21).ToArray().Length); } } } Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/MultiProducerRequestTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/MultiProducerRequestTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Request/MultiProducerRequestTests.cs (working copy) @@ -54,10 +54,10 @@ request.WriteTo(ms); byte[] bytes = ms.ToArray(); Assert.IsNotNull(bytes); - Assert.AreEqual(152, bytes.Length); + Assert.AreEqual(156, bytes.Length); // first 4 bytes = the length of the request - Assert.AreEqual(148, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray()), 0)); + Assert.AreEqual(152, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray()), 0)); // next 2 bytes = the RequestType which in this case should be Produce Assert.AreEqual((short)RequestTypes.MultiProduce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray()), 0)); Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Kafka.Client.Tests.csproj =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Kafka.Client.Tests.csproj (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/Kafka.Client.Tests.csproj (working copy) @@ -79,10 +79,12 @@ ..\..\..\..\lib\nunit\2.5.9\nunit.framework.dll + + @@ -103,6 +105,14 @@ + + + Settings.StyleCop + + + Designer + + - \ No newline at end of file + Index: clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageTests.cs =================================================================== --- clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageTests.cs (revision 1184850) +++ clients/csharp/src/Kafka/Tests/Kafka.Client.Tests/MessageTests.cs (working copy) @@ -34,8 +34,8 @@ private readonly int ChecksumPartLength = 4; private readonly int MagicNumberPartOffset = 0; - private readonly int ChecksumPartOffset = 1; - private readonly int DataPartOffset = 5; + private readonly int ChecksumPartOffset = 2; + private readonly int DataPartOffset = 6; /// /// Demonstrates a properly parsed message. @@ -46,16 +46,18 @@ Crc32Hasher crc32 = new Crc32Hasher(); string payload = "kafka"; - byte magic = 0; + byte magic = 1; + byte attributes = 0; byte[] payloadData = Encoding.UTF8.GetBytes(payload); byte[] payloadSize = BitConverter.GetBytes(payloadData.Length); byte[] checksum = crc32.ComputeHash(payloadData); - byte[] messageData = new byte[payloadData.Length + 1 + payloadSize.Length + checksum.Length]; + byte[] messageData = new byte[payloadData.Length + 2 + payloadSize.Length + checksum.Length]; Buffer.BlockCopy(payloadSize, 0, messageData, 0, payloadSize.Length); messageData[4] = magic; - Buffer.BlockCopy(checksum, 0, messageData, payloadSize.Length + 1, checksum.Length); - Buffer.BlockCopy(payloadData, 0, messageData, payloadSize.Length + 1 + checksum.Length, payloadData.Length); + messageData[5] = attributes; + Buffer.BlockCopy(checksum, 0, messageData, payloadSize.Length + 2, checksum.Length); + Buffer.BlockCopy(payloadData, 0, messageData, payloadSize.Length + 2 + checksum.Length, payloadData.Length); Message message = Message.ParseFrom(messageData); @@ -71,22 +73,25 @@ [Test] public void GetBytesValidSequence() { - Message message = new Message(new byte[10], (byte)245); + Message message = new Message(new byte[10], CompressionCodecs.NoCompressionCodec); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); // len(payload) + 1 + 4 - Assert.AreEqual(15, ms.Length); + Assert.AreEqual(16, ms.Length); // first 4 bytes = the magic number - Assert.AreEqual((byte)245, ms.ToArray()[0]); + Assert.AreEqual((byte)1, ms.ToArray()[0]); + // attributes + Assert.AreEqual((byte)0, ms.ToArray()[1]); + // next 4 bytes = the checksum - Assert.IsTrue(message.Checksum.SequenceEqual(ms.ToArray().Skip(1).Take(4).ToArray())); + Assert.IsTrue(message.Checksum.SequenceEqual(ms.ToArray().Skip(2).Take(4).ToArray())); // remaining bytes = the payload - Assert.AreEqual(10, ms.ToArray().Skip(5).ToArray().Length); + Assert.AreEqual(10, ms.ToArray().Skip(6).ToArray().Length); } [Test] @@ -97,12 +102,14 @@ MemoryStream ms = new MemoryStream(); message.WriteTo(ms); - Assert.AreEqual(0, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 0 + Assert.AreEqual(1, ms.ToArray()[MagicNumberPartOffset]); // default magic number should be 1 byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Assert.AreEqual(Crc32Hasher.Compute(messageBytes), checksumPart); + message.ToString(); + byte[] dataPart = new byte[messageBytes.Length]; Array.Copy(ms.ToArray(), DataPartOffset, dataPart, 0, messageBytes.Length); Assert.AreEqual(messageBytes, dataPart); @@ -113,11 +120,11 @@ { byte[] messageBytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; byte[] customChecksum = new byte[] { 3, 4, 5, 6 }; - Message message = new Message(messageBytes, (byte)33, customChecksum); + Message message = new Message(messageBytes, customChecksum); MemoryStream ms = new MemoryStream(); message.WriteTo(ms); - Assert.AreEqual((byte)33, ms.ToArray()[MagicNumberPartOffset]); + Assert.AreEqual((byte)1, ms.ToArray()[MagicNumberPartOffset]); byte[] checksumPart = new byte[ChecksumPartLength]; Array.Copy(ms.ToArray(), ChecksumPartOffset, checksumPart, 0, ChecksumPartLength); Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperAwareKafkaClientBase.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperAwareKafkaClientBase.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperAwareKafkaClientBase.cs (working copy) @@ -28,7 +28,7 @@ /// Initializes a new instance of the class. /// /// The config. - protected ZooKeeperAwareKafkaClientBase(ZKConfig config) + protected ZooKeeperAwareKafkaClientBase(ZooKeeperConfiguration config) { this.IsZooKeeperEnabled = config != null && !string.IsNullOrEmpty(config.ZkConnect); } Index: clients/csharp/src/Kafka/Kafka.Client/Exceptions/ConsumerTimeoutException.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Exceptions/ConsumerTimeoutException.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Exceptions/ConsumerTimeoutException.cs (working copy) @@ -16,6 +16,9 @@ */ using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Kafka.Client.Exceptions { Index: clients/csharp/src/Kafka/Kafka.Client/Kafka.Client.csproj =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Kafka.Client.csproj (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Kafka.Client.csproj (working copy) @@ -109,17 +109,19 @@ - - - - - + + + + + + - - - - + + + + + @@ -135,7 +137,11 @@ + + + + @@ -145,7 +151,12 @@ Code + + + + + @@ -153,8 +164,8 @@ - - + + @@ -224,7 +235,11 @@ - + + + Settings.StyleCop + + - \ No newline at end of file + Index: clients/csharp/src/Kafka/Kafka.Client/Cfg/ISyncProducerConfigShared.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Cfg/ISyncProducerConfigShared.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Cfg/ISyncProducerConfigShared.cs (working copy) @@ -25,8 +25,6 @@ int SocketTimeout { get; set; } - int ReconnectInterval { get; set; } - int MaxMessageSize { get; set; } } } Index: clients/csharp/src/Kafka/Kafka.Client/Cfg/IAsyncProducerConfigShared.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Cfg/IAsyncProducerConfigShared.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Cfg/IAsyncProducerConfigShared.cs (working copy) @@ -17,24 +17,10 @@ namespace Kafka.Client.Cfg { - using System.Collections.Generic; - internal interface IAsyncProducerConfigShared { - int QueueTime { get; set; } - - int QueueSize { get; set; } - - int BatchSize { get; set; } - string SerializerClass { get; set; } - string CallbackHandler { get; set; } - - string EventHandler { get; set; } - - IDictionary CallbackHandlerProps { get; set; } - - IDictionary EventHandlerProps { get; set; } + string CallbackHandlerClass { get; set; } } } Index: clients/csharp/src/Kafka/Kafka.Client/Utils/Guard.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Utils/Guard.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Utils/Guard.cs (working copy) @@ -18,32 +18,56 @@ namespace Kafka.Client.Utils { using System; + using System.Collections; using System.Collections.Generic; using System.Linq.Expressions; using System.Text.RegularExpressions; internal static class Guard { - /// - /// Checks whether given expression is true. Throws if not. - /// - /// - /// The assertion. - /// - /// - /// Thrown when condition is not met. - /// - public static void Assert(Expression> assertion) + public static void NotNull(object parameter, string paramName) { - var compiled = assertion.Compile(); - var evaluatedValue = compiled(); - if (!evaluatedValue) + if (parameter == null) { - throw new InvalidOperationException( - string.Format("'{0}' is not met.", Normalize(assertion.ToString()))); + throw new ArgumentNullException(paramName); } } + public static void Count(ICollection parameter, int length, string paramName) + { + if (parameter.Count != length) + { + throw new ArgumentOutOfRangeException(paramName, parameter.Count, string.Empty); + } + } + + public static void Greater(int parameter, int expected, string paramName) + { + if (parameter <= expected) + { + throw new ArgumentOutOfRangeException(paramName, parameter, string.Empty); + } + } + + public static void NotNullNorEmpty(string parameter, string paramName) + { + if (string.IsNullOrEmpty(parameter)) + { + throw new ArgumentException("Given string is empty", paramName); + } + } + + public static void AllNotNull(IEnumerable parameter, string paramName) + { + foreach (var par in parameter) + { + if (par == null) + { + throw new ArgumentNullException(paramName); + } + } + } + /// /// Checks whether given expression is true. Throws given exception type if not. /// Index: clients/csharp/src/Kafka/Kafka.Client/Utils/ReflectionHelper.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Utils/ReflectionHelper.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Utils/ReflectionHelper.cs (working copy) @@ -25,23 +25,21 @@ public static T Instantiate(string className) where T : class { - Type t1; object o1; if (string.IsNullOrEmpty(className)) { return default(T); } - if (className.Contains("`1")) + Type t1 = Type.GetType(className, true); + if (t1.IsGenericType) { - t1 = Type.GetType(className); var t2 = typeof(T).GetGenericArguments(); var t3 = t1.MakeGenericType(t2); o1 = Activator.CreateInstance(t3); return o1 as T; } - t1 = Type.GetType(className, true); o1 = Activator.CreateInstance(t1); return o1 as T; } Index: clients/csharp/src/Kafka/Kafka.Client/KafkaConnection.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/KafkaConnection.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/KafkaConnection.cs (working copy) @@ -23,6 +23,7 @@ using System.Threading; using Kafka.Client.Producers.Async; using Kafka.Client.Requests; + using Kafka.Client.Serialization; using Kafka.Client.Utils; /// @@ -30,9 +31,10 @@ /// public class KafkaConnection : IDisposable { - /// - /// TCP client that connects to the server. - /// + private readonly int bufferSize; + + private readonly int socketTimeout; + private readonly TcpClient client; private volatile bool disposed; @@ -42,104 +44,33 @@ /// /// The server to connect to. /// The port to connect to. - public KafkaConnection(string server, int port) + public KafkaConnection(string server, int port, int bufferSize, int socketTimeout) { - Server = server; - Port = port; + this.bufferSize = bufferSize; + this.socketTimeout = socketTimeout; // connection opened - client = new TcpClient(server, port); + this.client = new TcpClient(server, port) + { + ReceiveTimeout = socketTimeout, + SendTimeout = socketTimeout, + ReceiveBufferSize = bufferSize, + SendBufferSize = bufferSize + }; + var stream = this.client.GetStream(); + this.Reader = new KafkaBinaryReader(stream); } - /// - /// Gets the server to which the connection is to be established. - /// - public string Server { get; private set; } - - /// - /// Gets the port to which the connection is to be established. - /// - public int Port { get; private set; } + public KafkaBinaryReader Reader { get; private set; } /// - /// Readds data from the server. - /// - /// - /// Defauls the amount of time that a read operation blocks waiting for data to . - /// - /// The number of bytes to read from the server. - /// The data read from the server as a byte array. - public byte[] Read(int size) - { - this.EnsuresNotDisposed(); - return Read(size, Timeout.Infinite); - } - - /// - /// Readds data from the server. - /// - /// The number of bytes to read from the server. - /// The amount of time that a read operation blocks waiting for data. - /// The data read from the server as a byte array. - public byte[] Read(int size, int readTimeout) - { - this.EnsuresNotDisposed(); - byte[] bytes; - NetworkStream stream = client.GetStream(); - stream.ReadTimeout = readTimeout; - int numberOfTries = 0; - - int readSize = size; - if (client.ReceiveBufferSize < size) - { - readSize = client.ReceiveBufferSize; - } - - using (var ms = new MemoryStream()) - { - var bytesToRead = new byte[client.ReceiveBufferSize]; - - while (true) - { - int numberOfBytesRead = stream.Read(bytesToRead, 0, readSize); - if (numberOfBytesRead > 0) - { - ms.Write(bytesToRead, 0, numberOfBytesRead); - } - - if (ms.Length >= size) - { - break; - } - - if (numberOfBytesRead == 0) - { - if (numberOfTries >= 1000) - { - break; - } - - numberOfTries++; - Thread.Sleep(10); - } - } - - bytes = new byte[ms.Length]; - ms.Seek(0, SeekOrigin.Begin); - ms.Read(bytes, 0, (int)ms.Length); - } - - return bytes; - } - - /// /// Writes a producer request to the server asynchronously. /// /// The request to make. public void BeginWrite(ProducerRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); + Guard.NotNull(request, "request"); NetworkStream stream = client.GetStream(); byte[] data = request.RequestBuffer.GetBuffer(); stream.BeginWrite(data, 0, data.Length, asyncResult => ((NetworkStream)asyncResult.AsyncState).EndWrite(asyncResult), stream); @@ -157,7 +88,7 @@ public void BeginWrite(ProducerRequest request, MessageSent callback) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); + Guard.NotNull(request, "request"); if (callback == null) { this.BeginWrite(request); @@ -191,8 +122,8 @@ public void Write(ProducerRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); - this.Write(request.RequestBuffer.GetBuffer(), Timeout.Infinite); + Guard.NotNull(request, "request"); + this.Write(request.RequestBuffer.GetBuffer()); } /// @@ -205,21 +136,18 @@ public void Write(MultiProducerRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); - this.Write(request.RequestBuffer.GetBuffer(), Timeout.Infinite); + Guard.NotNull(request, "request"); + this.Write(request.RequestBuffer.GetBuffer()); } /// /// Writes data to the server. /// /// The data to write to the server. - /// The amount of time that a write operation blocks waiting for data. - private void Write(byte[] data, int writeTimeout) + private void Write(byte[] data) { NetworkStream stream = this.client.GetStream(); - stream.WriteTimeout = writeTimeout; - - // Send the message to the connected TcpServer. + //// Send the message to the connected TcpServer. stream.Write(data, 0, data.Length); } @@ -233,8 +161,8 @@ public void Write(FetchRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); - this.Write(request.RequestBuffer.GetBuffer(), Timeout.Infinite); + Guard.NotNull(request, "request"); + this.Write(request.RequestBuffer.GetBuffer()); } /// @@ -247,8 +175,8 @@ public void Write(MultiFetchRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); - this.Write(request.RequestBuffer.GetBuffer(), Timeout.Infinite); + Guard.NotNull(request, "request"); + this.Write(request.RequestBuffer.GetBuffer()); } /// @@ -261,8 +189,8 @@ public void Write(OffsetRequest request) { this.EnsuresNotDisposed(); - Guard.Assert(() => request != null); - this.Write(request.RequestBuffer.GetBuffer(), Timeout.Infinite); + Guard.NotNull(request, "request"); + this.Write(request.RequestBuffer.GetBuffer()); } /// @@ -278,7 +206,6 @@ this.disposed = true; if (this.client != null) { - this.client.GetStream().Close(); this.client.Close(); } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/ProducerPool.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/ProducerPool.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/ProducerPool.cs (working copy) @@ -33,6 +33,8 @@ internal abstract class ProducerPool : IProducerPool where TData : class { + protected bool Disposed { get; set; } + /// /// Factory method used to instantiating either, /// synchronous or asynchronous, producer pool based on configuration. @@ -46,7 +48,7 @@ /// /// Instantiated either, synchronous or asynchronous, producer pool /// - public static ProducerPool CreatePool(ProducerConfig config, IEncoder serializer) + public static ProducerPool CreatePool(ProducerConfiguration config, IEncoder serializer) { if (config.ProducerType == ProducerTypes.Async) { @@ -78,7 +80,7 @@ /// Instantiated either, synchronous or asynchronous, producer pool /// public static ProducerPool CreatePool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, ICallbackHandler cbkHandler) { @@ -104,11 +106,11 @@ /// Should be used for testing purpose only /// protected ProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer) { - Guard.Assert(() => config != null); - Guard.Assert(() => serializer != null); + Guard.NotNull(config, "config"); + Guard.NotNull(serializer, "serializer"); this.Config = config; this.Serializer = serializer; @@ -127,19 +129,19 @@ /// The callback invoked after new broker is added. /// protected ProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, ICallbackHandler callbackHandler) { - Guard.Assert(() => config != null); - Guard.Assert(() => serializer != null); + Guard.NotNull(config, "config"); + Guard.NotNull(serializer, "serializer"); this.Config = config; this.Serializer = serializer; this.CallbackHandler = callbackHandler; } - protected ProducerConfig Config { get; private set; } + protected ProducerConfiguration Config { get; private set; } protected IEncoder Serializer { get; private set; } @@ -162,7 +164,9 @@ /// public void Send(ProducerPoolData poolData) { - Guard.Assert(() => poolData != null); + this.EnsuresNotDisposed(); + Guard.NotNull(poolData, "poolData"); + this.Send(new[] { poolData }); } @@ -176,5 +180,27 @@ /// Used for multi-topic request /// public abstract void Send(IEnumerable> poolData); + + /// + /// Releases all unmanaged and managed resources + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected abstract void Dispose(bool disposing); + + /// + /// Ensures that object was not disposed + /// + protected void EnsuresNotDisposed() + { + if (this.Disposed) + { + throw new ObjectDisposedException(this.GetType().Name); + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.StrMsg.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.StrMsg.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.StrMsg.cs (working copy) @@ -40,7 +40,7 @@ /// /// Should be used for testing purpose only. /// - internal Producer(ProducerConfig config, IPartitioner partitioner, IProducerPool producerPool, bool populateProducerPool) + internal Producer(ProducerConfiguration config, IPartitioner partitioner, IProducerPool producerPool, bool populateProducerPool) : base(config, partitioner, producerPool, populateProducerPool) { } @@ -53,7 +53,7 @@ /// Can be used when all config parameters will be specified through the config object /// and will be instantiated via reflection /// - public Producer(ProducerConfig config) + public Producer(ProducerConfiguration config) : base(config) { } @@ -71,7 +71,7 @@ /// Can be used to provide pre-instantiated objects for all config parameters /// that would otherwise be instantiated via reflection. /// - public Producer(ProducerConfig config, IPartitioner partitioner, IEncoder encoder, ICallbackHandler callbackHandler) + public Producer(ProducerConfiguration config, IPartitioner partitioner, IEncoder encoder, ICallbackHandler callbackHandler) : base(config, partitioner, encoder, callbackHandler) { } @@ -88,7 +88,7 @@ /// Can be used to provide pre-instantiated objects for all config parameters /// that would otherwise be instantiated via reflection. /// - public Producer(ProducerConfig config, IPartitioner partitioner, IEncoder encoder) + public Producer(ProducerConfiguration config, IPartitioner partitioner, IEncoder encoder) : base(config, partitioner, encoder) { } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ZKBrokerPartitionInfo.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ZKBrokerPartitionInfo.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ZKBrokerPartitionInfo.cs (working copy) @@ -18,6 +18,7 @@ namespace Kafka.Client.Producers.Partitioning { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.Reflection; @@ -66,8 +67,8 @@ /// /// The config. /// The callback invoked when new broker is added. - public ZKBrokerPartitionInfo(ZKConfig config, Action callback) - : this(new ZooKeeperClient(config.ZkConnect, config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) + public ZKBrokerPartitionInfo(ProducerConfiguration config, Action callback) + : this(new ZooKeeperClient(config.ZooKeeper.ZkConnect, config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer)) { this.callback = callback; } @@ -93,7 +94,7 @@ /// public SortedSet GetBrokerPartitionInfo(string topic) { - Guard.Assert(() => !string.IsNullOrEmpty(topic)); + Guard.NotNullNorEmpty(topic, "topic"); this.EnsuresNotDisposed(); SortedSet brokerPartitions = null; @@ -101,11 +102,15 @@ { brokerPartitions = this.topicBrokerPartitions[topic]; } + else + { + this.topicBrokerPartitions.Add(topic, null); + } if (brokerPartitions == null || brokerPartitions.Count == 0) { var numBrokerPartitions = this.BootstrapWithExistingBrokers(topic); - this.topicBrokerPartitions.Add(topic, numBrokerPartitions); + this.topicBrokerPartitions[topic] = numBrokerPartitions; return numBrokerPartitions; } @@ -168,7 +173,7 @@ return; } - this.brokers = new Dictionary(); + this.brokers = new ConcurrentDictionary(); IList brokerIds = this.zkclient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerIdsPath); foreach (var brokerId in brokerIds) { @@ -191,7 +196,7 @@ return; } - this.topicBrokerPartitions = new Dictionary>(); + this.topicBrokerPartitions = new ConcurrentDictionary>(); this.zkclient.MakeSurePersistentPathExists(ZooKeeperClient.DefaultBrokerTopicsPath); IList topics = this.zkclient.GetChildrenParentMayNotExist(ZooKeeperClient.DefaultBrokerTopicsPath); foreach (string topic in topics) @@ -306,7 +311,7 @@ /// public void HandleStateChanged(ZooKeeperStateChangedEventArgs args) { - Guard.Assert(() => args != null); + Guard.NotNull(args, "args"); Guard.Assert(() => args.State != KeeperState.Unknown); this.EnsuresNotDisposed(); @@ -322,7 +327,7 @@ /// public void HandleSessionCreated(ZooKeeperSessionCreatedEventArgs args) { - Guard.Assert(() => args != null); + Guard.NotNull(args, "args"); this.EnsuresNotDisposed(); Logger.Debug("ZK expired; release old list of broker partitions for topics "); Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ConfigBrokerPartitionInfo.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ConfigBrokerPartitionInfo.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/ConfigBrokerPartitionInfo.cs (working copy) @@ -32,16 +32,16 @@ /// internal class ConfigBrokerPartitionInfo : IBrokerPartitionInfo { - private readonly ProducerConfig config; + private readonly ProducerConfiguration config; private IDictionary brokers; /// /// Initializes a new instance of the class. /// /// The config. - public ConfigBrokerPartitionInfo(ProducerConfig config) + public ConfigBrokerPartitionInfo(ProducerConfiguration config) { - Guard.Assert(() => config != null); + Guard.NotNull(config, "config"); this.config = config; this.InitializeBrokers(); } @@ -67,7 +67,7 @@ /// Partition ID would be allways 0 public SortedSet GetBrokerPartitionInfo(string topic) { - Guard.Assert(() => !string.IsNullOrEmpty(topic)); + Guard.NotNullNorEmpty(topic, "topic"); var partitions = new SortedSet(); foreach (var item in this.brokers) { @@ -108,13 +108,11 @@ } this.brokers = new Dictionary(); - string[] brokersInfoList = this.config.BrokerPartitionInfo.Split(','); - foreach (string item in brokersInfoList) + foreach (var item in this.config.Brokers) { - var parts = item.Split(':'); - int id = int.Parse(parts[0], CultureInfo.InvariantCulture); - int port = int.Parse(parts[2], CultureInfo.InvariantCulture); - this.brokers.Add(id, new Broker(id, parts[1], parts[1], port)); + this.brokers.Add( + item.BrokerId, + new Broker(item.BrokerId, item.Host, item.Host, item.Port)); } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/DefaultPartitioner.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/DefaultPartitioner.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Partitioning/DefaultPartitioner.cs (working copy) @@ -41,7 +41,7 @@ /// public int Partition(TKey key, int numPartitions) { - Guard.Assert(() => numPartitions > 0); + Guard.Greater(numPartitions, 0, "numPartitions"); return key == null ? Randomizer.Next(numPartitions) : key.GetHashCode() % numPartitions; Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Producer.cs (working copy) @@ -41,13 +41,13 @@ /// Provides serialization of data through a user-specified encoder, zookeeper based automatic broker discovery /// and software load balancing through an optionally user-specified partitioner /// - public class Producer : ZooKeeperAwareKafkaClientBase, IProducer + public class Producer : KafkaClientBase, IProducer where TKey : class where TData : class { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private static readonly Random Randomizer = new Random(); - private readonly ProducerConfig config; + private readonly ProducerConfiguration config; private readonly IProducerPool producerPool; private readonly IPartitioner partitioner; private readonly bool populateProducerPool; @@ -67,19 +67,19 @@ /// Should be used for testing purpose only. /// internal Producer( - ProducerConfig config, + ProducerConfiguration config, IPartitioner partitioner, IProducerPool producerPool, bool populateProducerPool = true) - : base(config) { - Guard.Assert(() => config != null); - Guard.Assert(() => producerPool != null); + Guard.NotNull(config, "config"); + Guard.NotNull(producerPool, "producerPool"); + this.config = config; this.partitioner = partitioner ?? new DefaultPartitioner(); this.populateProducerPool = populateProducerPool; this.producerPool = producerPool; - if (this.IsZooKeeperEnabled) + if (this.config.IsZooKeeperEnabled) { this.brokerPartitionInfo = new ZKBrokerPartitionInfo(this.config, this.Callback); } @@ -107,13 +107,13 @@ /// Can be used when all config parameters will be specified through the config object /// and will be instantiated via reflection /// - public Producer(ProducerConfig config) + public Producer(ProducerConfiguration config) : this( config, ReflectionHelper.Instantiate>(config.PartitionerClass), ProducerPool.CreatePool(config, ReflectionHelper.Instantiate>(config.SerializerClass))) { - Guard.Assert(() => config != null); + Guard.NotNull(config, "config"); } /// @@ -131,7 +131,7 @@ /// that would otherwise be instantiated via reflection. /// public Producer( - ProducerConfig config, + ProducerConfiguration config, IPartitioner partitioner, IEncoder encoder, ICallbackHandler callbackHandler) @@ -140,8 +140,8 @@ partitioner, ProducerPool.CreatePool(config, encoder, callbackHandler)) { - Guard.Assert(() => config != null); - Guard.Assert(() => encoder != null); + Guard.NotNull(config, "config"); + Guard.NotNull(encoder, "encoder"); } /// @@ -157,7 +157,7 @@ /// that would otherwise be instantiated via reflection. /// public Producer( - ProducerConfig config, + ProducerConfiguration config, IPartitioner partitioner, IEncoder encoder) : this( @@ -165,8 +165,8 @@ partitioner, ProducerPool.CreatePool(config, encoder, null)) { - Guard.Assert(() => config != null); - Guard.Assert(() => encoder != null); + Guard.NotNull(config, "config"); + Guard.NotNull(encoder, "encoder"); } /// @@ -176,8 +176,9 @@ /// The producer data objects that encapsulate the topic, key and message data. public void Send(IEnumerable> data) { - Guard.Assert(() => data != null); - Guard.Assert(() => data.Count() > 0); + Guard.NotNull(data, "data"); + Guard.Greater(data.Count(), 0, "data"); + this.EnsuresNotDisposed(); var poolRequests = new List>(); foreach (var dataItem in data) @@ -197,10 +198,11 @@ /// The producer data object that encapsulates the topic, key and message data. public void Send(ProducerData data) { - Guard.Assert(() => data != null); - Guard.Assert(() => !string.IsNullOrEmpty(data.Topic)); - Guard.Assert(() => data.Data != null); - Guard.Assert(() => data.Data.Count() > 0); + Guard.NotNull(data, "data"); + Guard.NotNullNorEmpty(data.Topic, "data.Topic"); + Guard.NotNull(data.Data, "data.Data"); + Guard.Greater(data.Data.Count(), 0, "data.Data"); + this.EnsuresNotDisposed(); this.Send(new[] { data }); } @@ -233,6 +235,11 @@ { this.brokerPartitionInfo.Dispose(); } + + if (this.producerPool != null) + { + this.producerPool.Dispose(); + } } catch (Exception exc) { @@ -249,8 +256,8 @@ /// The broker port. private void Callback(int bid, string host, int port) { - Guard.Assert(() => !string.IsNullOrEmpty(host)); - Guard.Assert(() => port > 0); + Guard.NotNullNorEmpty(host, "host"); + Guard.Greater(port, 0, "port"); if (this.populateProducerPool) { @@ -270,7 +277,7 @@ /// Partition Id private int GetPartitionId(TKey key, int numPartitions) { - Guard.Assert(() => numPartitions > 0); + Guard.Greater(numPartitions, 0, "numPartitions"); return key == null ? Randomizer.Next(numPartitions) : this.partitioner.Partition(key, numPartitions); @@ -297,7 +304,7 @@ int partitionId = this.GetPartitionId(dataItem.Key, totalNumPartitions); Partition brokerIdPartition = brokerPartitions.ToList()[partitionId]; Broker brokerInfo = this.brokerPartitionInfo.GetBrokerInfo(brokerIdPartition.BrokerId); - if (this.IsZooKeeperEnabled) + if (this.config.IsZooKeeperEnabled) { Logger.DebugFormat( CultureInfo.CurrentCulture, Index: clients/csharp/src/Kafka/Kafka.Client/Producers/IProducerPool.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/IProducerPool.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/IProducerPool.cs (working copy) @@ -17,6 +17,7 @@ namespace Kafka.Client.Producers { + using System; using System.Collections.Generic; using Kafka.Client.Cluster; @@ -24,7 +25,7 @@ /// Pool of producers used by producer high-level API /// /// The type of the data. - internal interface IProducerPool + internal interface IProducerPool : IDisposable { /// /// Selects either a synchronous or an asynchronous producer, for Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducer.cs (working copy) @@ -30,15 +30,14 @@ /// public class SyncProducer : ISyncProducer { - private readonly SyncProducerConfig config; + private readonly KafkaConnection connection; + private volatile bool disposed; + /// /// Gets producer config /// - public SyncProducerConfig Config - { - get { return config; } - } + public SyncProducerConfiguration Config { get; private set; } /// /// Initializes a new instance of the class. @@ -46,10 +45,15 @@ /// /// The producer config. /// - public SyncProducer(SyncProducerConfig config) + public SyncProducer(SyncProducerConfiguration config) { - Guard.Assert(() => config != null); - this.config = config; + Guard.NotNull(config, "config"); + this.Config = config; + this.connection = new KafkaConnection( + this.Config.Host, + this.Config.Port, + config.BufferSize, + config.SocketTimeout); } /// @@ -66,14 +70,13 @@ /// public void Send(string topic, int partition, IEnumerable messages) { - Guard.Assert(() => !string.IsNullOrEmpty(topic)); - Guard.Assert(() => messages != null); - Guard.Assert( - () => messages.All(x => x != null)); + Guard.NotNullNorEmpty(topic, "topic"); + Guard.NotNull(messages, "messages"); + Guard.AllNotNull(messages, "messages.items"); Guard.Assert( () => messages.All( x => x.PayloadSize <= this.Config.MaxMessageSize)); - + this.EnsuresNotDisposed(); this.Send(new ProducerRequest(topic, partition, messages)); } @@ -85,11 +88,8 @@ /// public void Send(ProducerRequest request) { - Guard.Assert(() => request != null); - using (var conn = new KafkaConnection(this.config.Host, this.config.Port)) - { - conn.Write(request); - } + this.EnsuresNotDisposed(); + this.connection.Write(request); } /// @@ -100,7 +100,7 @@ /// public void MultiSend(IEnumerable requests) { - Guard.Assert(() => requests != null); + Guard.NotNull(requests, "requests"); Guard.Assert( () => requests.All( x => x != null && x.MessageSet != null && x.MessageSet.Messages != null)); @@ -108,12 +108,48 @@ () => requests.All( x => x.MessageSet.Messages.All( y => y != null && y.PayloadSize <= this.Config.MaxMessageSize))); + this.EnsuresNotDisposed(); + var multiRequest = new MultiProducerRequest(requests); + this.connection.Write(multiRequest); + } - var multiRequest = new MultiProducerRequest(requests); - using (var conn = new KafkaConnection(this.config.Host, this.config.Port)) + /// + /// Releases all unmanaged and managed resources + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposing) { - conn.Write(multiRequest); + return; } + + if (this.disposed) + { + return; + } + + this.disposed = true; + if (this.connection != null) + { + this.connection.Dispose(); + } } + + /// + /// Ensures that object was not disposed + /// + private void EnsuresNotDisposed() + { + if (this.disposed) + { + throw new ObjectDisposedException(this.GetType().Name); + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/ISyncProducer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/ISyncProducer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/ISyncProducer.cs (working copy) @@ -17,6 +17,7 @@ namespace Kafka.Client.Producers.Sync { + using System; using System.Collections.Generic; using Kafka.Client.Messages; using Kafka.Client.Requests; @@ -24,7 +25,7 @@ /// /// Sends messages encapsulated in request to Kafka server synchronously /// - public interface ISyncProducer + public interface ISyncProducer : IDisposable { /// /// Constructs producer request and sends it to given broker partition synchronously Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducerPool.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducerPool.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Sync/SyncProducerPool.cs (working copy) @@ -40,6 +40,7 @@ { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private readonly IDictionary syncProducers; + private volatile bool disposed; /// /// Factory method used to instantiating synchronous producer pool @@ -53,7 +54,7 @@ /// /// Instantiated synchronous producer pool /// - public static SyncProducerPool CreateSyncPool(ProducerConfig config, IEncoder serializer) + public static SyncProducerPool CreateSyncPool(ProducerConfiguration config, IEncoder serializer) { return new SyncProducerPool(config, serializer); } @@ -73,7 +74,7 @@ /// /// Instantiated synchronous producer pool /// - public static SyncProducerPool CreateSyncPool(ProducerConfig config, IEncoder serializer, ICallbackHandler callbackHandler) + public static SyncProducerPool CreateSyncPool(ProducerConfiguration config, IEncoder serializer, ICallbackHandler callbackHandler) { return new SyncProducerPool(config, serializer, callbackHandler); } @@ -97,7 +98,7 @@ /// Should be used for testing purpose only /// private SyncProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, IDictionary syncProducers, ICallbackHandler cbkHandler) @@ -122,7 +123,7 @@ /// Should be used for testing purpose only /// private SyncProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, ICallbackHandler cbkHandler) : this(config, serializer, new Dictionary(), cbkHandler) @@ -138,12 +139,12 @@ /// /// The serializer. /// - private SyncProducerPool(ProducerConfig config, IEncoder serializer) + private SyncProducerPool(ProducerConfiguration config, IEncoder serializer) : this( config, serializer, new Dictionary(), - ReflectionHelper.Instantiate(config.CallbackHandler)) + ReflectionHelper.Instantiate(config.CallbackHandlerClass)) { } @@ -158,7 +159,8 @@ /// public override void Send(IEnumerable> poolData) { - Guard.Assert(() => poolData != null); + this.EnsuresNotDisposed(); + Guard.NotNull(poolData, "poolData"); Dictionary>> distinctBrokers = poolData.GroupBy( x => x.BidPid.BrokerId, x => x) .ToDictionary(x => x.Key, x => x.ToList()); @@ -188,15 +190,10 @@ /// The broker informations. public override void AddProducer(Broker broker) { - Guard.Assert(() => broker != null); - var syncConfig = new SyncProducerConfig - { - Host = broker.Host, - Port = broker.Port, - BufferSize = this.Config.BufferSize, - ConnectTimeout = this.Config.ConnectTimeout, - ReconnectInterval = this.Config.ReconnectInterval - }; + this.EnsuresNotDisposed(); + Guard.NotNull(broker, "broker"); + + var syncConfig = new SyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port); var syncProducer = new SyncProducer(syncConfig); Logger.InfoFormat( CultureInfo.CurrentCulture, @@ -206,5 +203,24 @@ broker.Port); this.syncProducers.Add(broker.Id, syncProducer); } + + protected override void Dispose(bool disposing) + { + if (!disposing) + { + return; + } + + if (this.disposed) + { + return; + } + + this.disposed = true; + foreach (var syncProducer in this.syncProducers.Values) + { + syncProducer.Dispose(); + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducerPool.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducerPool.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducerPool.cs (working copy) @@ -39,6 +39,7 @@ { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); private readonly IDictionary asyncProducers; + private volatile bool disposed; /// /// Factory method used to instantiating asynchronous producer pool @@ -56,7 +57,7 @@ /// Instantiated asynchronous producer pool /// public static AsyncProducerPool CreateAsyncPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, ICallbackHandler cbkHandler) { @@ -76,7 +77,7 @@ /// Instantiated asynchronous producer pool /// public static AsyncProducerPool CreateAsyncPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer) { return new AsyncProducerPool(config, serializer); @@ -101,7 +102,7 @@ /// Should be used for testing purpose only /// private AsyncProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, IDictionary asyncProducers, ICallbackHandler cbkHandler) @@ -123,7 +124,7 @@ /// The callback invoked after new broker is added. /// private AsyncProducerPool( - ProducerConfig config, + ProducerConfiguration config, IEncoder serializer, ICallbackHandler cbkHandler) : this(config, serializer, new Dictionary(), cbkHandler) @@ -139,12 +140,12 @@ /// /// The serializer. /// - private AsyncProducerPool(ProducerConfig config, IEncoder serializer) + private AsyncProducerPool(ProducerConfiguration config, IEncoder serializer) : this( config, serializer, new Dictionary(), - ReflectionHelper.Instantiate(config.CallbackHandler)) + ReflectionHelper.Instantiate(config.CallbackHandlerClass)) { } @@ -159,7 +160,8 @@ /// public override void Send(IEnumerable> poolData) { - Guard.Assert(() => poolData != null); + this.EnsuresNotDisposed(); + Guard.NotNull(poolData, "poolData"); Dictionary>> distinctBrokers = poolData.GroupBy( x => x.BidPid.BrokerId, x => x) .ToDictionary(x => x.Key, x => x.ToList()); @@ -184,14 +186,10 @@ /// The broker informations. public override void AddProducer(Broker broker) { - Guard.Assert(() => broker != null); - var asyncConfig = new AsyncProducerConfig + this.EnsuresNotDisposed(); + Guard.NotNull(broker, "broker"); + var asyncConfig = new AsyncProducerConfiguration(this.Config, broker.Id, broker.Host, broker.Port) { - Host = broker.Host, - Port = broker.Port, - QueueTime = this.Config.QueueTime, - QueueSize = this.Config.QueueSize, - BatchSize = this.Config.BatchSize, SerializerClass = this.Config.SerializerClass }; var asyncProducer = new AsyncProducer(asyncConfig, this.CallbackHandler); @@ -203,5 +201,24 @@ broker.Port); this.asyncProducers.Add(broker.Id, asyncProducer); } + + protected override void Dispose(bool disposing) + { + if (!disposing) + { + return; + } + + if (this.disposed) + { + return; + } + + this.disposed = true; + foreach (var asyncProducer in this.asyncProducers.Values) + { + asyncProducer.Dispose(); + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Async/AsyncProducer.cs (working copy) @@ -28,19 +28,16 @@ /// /// Sends messages encapsulated in request to Kafka server asynchronously /// - public class AsyncProducer : IAsyncProducer, IDisposable + public class AsyncProducer : IAsyncProducer { - private readonly AsyncProducerConfig config; private readonly ICallbackHandler callbackHandler; - private KafkaConnection connection = null; + private readonly KafkaConnection connection; + private volatile bool disposed; /// /// Gets producer config /// - public AsyncProducerConfig Config - { - get { return config; } - } + public AsyncProducerConfiguration Config { get; private set; } /// /// Initializes a new instance of the class. @@ -48,10 +45,10 @@ /// /// The producer config. /// - public AsyncProducer(AsyncProducerConfig config) + public AsyncProducer(AsyncProducerConfiguration config) : this( config, - ReflectionHelper.Instantiate(config.CallbackHandler)) + ReflectionHelper.Instantiate(config.CallbackHandlerClass)) { } @@ -65,14 +62,18 @@ /// The callback invoked when a request is finished being sent. /// public AsyncProducer( - AsyncProducerConfig config, + AsyncProducerConfiguration config, ICallbackHandler callbackHandler) { - Guard.Assert(() => config != null); + Guard.NotNull(config, "config"); - this.config = config; + this.Config = config; this.callbackHandler = callbackHandler; - this.connection = new KafkaConnection(this.config.Host, this.config.Port); + this.connection = new KafkaConnection( + this.Config.Host, + this.Config.Port, + this.Config.BufferSize, + this.Config.SocketTimeout); } /// @@ -83,7 +84,8 @@ /// public void Send(ProducerRequest request) { - Guard.Assert(() => request != null); + this.EnsuresNotDisposed(); + Guard.NotNull(request, "request"); Guard.Assert(() => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize)); if (this.callbackHandler != null) { @@ -91,7 +93,7 @@ } else { - connection.BeginWrite(request); + this.connection.BeginWrite(request); } } @@ -106,12 +108,13 @@ /// public void Send(ProducerRequest request, MessageSent callback) { - Guard.Assert(() => request != null); - Guard.Assert(() => request.MessageSet != null); - Guard.Assert(() => request.MessageSet.Messages != null); + this.EnsuresNotDisposed(); + Guard.NotNull(request, "request"); + Guard.NotNull(request.MessageSet, "request.MessageSet"); + Guard.NotNull(request.MessageSet.Messages, "request.MessageSet.Messages"); Guard.Assert( () => request.MessageSet.Messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize)); - + connection.BeginWrite(request, callback); } @@ -129,10 +132,11 @@ /// public void Send(string topic, int partition, IEnumerable messages) { - Guard.Assert(() => !string.IsNullOrEmpty(topic)); - Guard.Assert(() => messages != null); + this.EnsuresNotDisposed(); + Guard.NotNullNorEmpty(topic, "topic"); + Guard.NotNull(messages, "messages"); Guard.Assert(() => messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize)); - + this.Send(new ProducerRequest(topic, partition, messages)); } @@ -153,19 +157,51 @@ /// public void Send(string topic, int partition, IEnumerable messages, MessageSent callback) { - Guard.Assert(() => !string.IsNullOrEmpty(topic)); - Guard.Assert(() => messages != null); + this.EnsuresNotDisposed(); + Guard.NotNullNorEmpty(topic, "topic"); + Guard.NotNull(messages, "messages"); Guard.Assert(() => messages.All(x => x.PayloadSize <= this.Config.MaxMessageSize)); - + this.Send(new ProducerRequest(topic, partition, messages), callback); } + /// + /// Releases all unmanaged and managed resources + /// public void Dispose() { - if (connection != null) + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposing) { - connection.Dispose(); + return; } + + if (this.disposed) + { + return; + } + + this.disposed = true; + if (this.connection != null) + { + this.connection.Dispose(); + } } + + /// + /// Ensures that object was not disposed + /// + private void EnsuresNotDisposed() + { + if (this.disposed) + { + throw new ObjectDisposedException(this.GetType().Name); + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Producers/Async/IAsyncProducer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Producers/Async/IAsyncProducer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Producers/Async/IAsyncProducer.cs (working copy) @@ -17,6 +17,7 @@ namespace Kafka.Client.Producers.Async { + using System; using System.Collections.Generic; using Kafka.Client.Messages; using Kafka.Client.Requests; @@ -24,7 +25,7 @@ /// /// Sends messages encapsulated in request to Kafka server asynchronously /// - public interface IAsyncProducer + public interface IAsyncProducer : IDisposable { /// /// Sends request to Kafka server asynchronously Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/IZooKeeperClient.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/IZooKeeperClient.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/IZooKeeperClient.cs (working copy) @@ -34,7 +34,7 @@ /// /// Used for testing purpose /// - int IdleTime { get; } + int? IdleTime { get; } /// /// Connects to ZooKeeper server within given time period and installs watcher in ZooKeeper Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperClient.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperClient.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperClient.cs (working copy) @@ -209,7 +209,8 @@ /// public bool WaitUntilConnected(int connectionTimeout) { - Guard.Assert(() => connectionTimeout > 0); + Guard.Greater(connectionTimeout, 0, "connectionTimeout"); + this.EnsuresNotDisposed(); if (this.eventWorker != null && this.eventWorker == Thread.CurrentThread) { @@ -250,7 +251,8 @@ /// public T RetryUntilConnected(Func callback) { - Guard.Assert(() => callback != null); + Guard.NotNull(callback, "callback"); + this.EnsuresNotDisposed(); if (this.zooKeeperEventWorker != null && this.zooKeeperEventWorker == Thread.CurrentThread) { @@ -290,7 +292,7 @@ /// public bool Exists(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); bool hasListeners = this.HasListeners(path); @@ -311,7 +313,7 @@ /// public bool Exists(string path, bool watch) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.RetryUntilConnected( @@ -332,7 +334,7 @@ /// public IList GetChildren(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); bool hasListeners = this.HasListeners(path); @@ -353,7 +355,7 @@ /// public IList GetChildren(string path, bool watch) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.RetryUntilConnected( @@ -375,7 +377,7 @@ /// public int CountChildren(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); try @@ -413,7 +415,7 @@ public T ReadData(string path, Stat stats, bool watch) where T : class { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); byte[] bytes = this.RetryUntilConnected( @@ -443,7 +445,7 @@ /// public T ReadData(string path, Stat stats) where T : class { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); bool hasListeners = this.HasListeners(path); @@ -461,7 +463,7 @@ /// public void WriteData(string path, object data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); this.WriteData(path, data, -1); @@ -484,7 +486,7 @@ /// public void WriteData(string path, object data, int expectedVersion) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); byte[] bytes = this.serializer.Serialize(data); @@ -507,7 +509,7 @@ /// public bool Delete(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.RetryUntilConnected( @@ -536,7 +538,7 @@ /// public bool DeleteRecursive(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); IList children; @@ -568,7 +570,7 @@ /// public void MakeSurePersistentPathExists(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); if (!this.Exists(path)) @@ -588,7 +590,7 @@ /// public IList GetChildrenParentMayNotExist(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); try @@ -616,7 +618,7 @@ public T ReadData(string path) where T : class { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.ReadData(path, false); @@ -671,8 +673,7 @@ /// public void CreatePersistent(string path, bool createParents) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); try { @@ -710,8 +711,8 @@ /// public void CreatePersistent(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); this.CreatePersistent(path, false); } @@ -730,8 +731,8 @@ /// public void CreatePersistent(string path, object data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); this.Create(path, data, CreateMode.Persistent); } @@ -753,8 +754,8 @@ /// public string CreatePersistentSequential(string path, object data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); return this.Create(path, data, CreateMode.PersistentSequential); } @@ -796,8 +797,8 @@ /// public void CreateEphemeral(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); this.Create(path, null, CreateMode.Ephemeral); } @@ -815,8 +816,8 @@ /// public void CreateEphemeral(string path, object data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); this.Create(path, data, CreateMode.Ephemeral); } @@ -837,8 +838,8 @@ /// public string CreateEphemeralSequential(string path, object data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); return this.Create(path, data, CreateMode.EphemeralSequential); } @@ -861,8 +862,8 @@ public T ReadData(string path, bool returnNullIfPathNotExists) where T : class { - Guard.Assert(() => !string.IsNullOrEmpty(path)); - + Guard.NotNullNorEmpty(path, "path"); + this.EnsuresNotDisposed(); try { return this.ReadData(path, null); Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperConnection.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperConnection.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperConnection.cs (working copy) @@ -130,7 +130,7 @@ /// public void Delete(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); this.Client.Delete(path, -1); @@ -150,7 +150,7 @@ /// public bool Exists(string path, bool watch) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.Client.Exists(path, true) != null; @@ -173,7 +173,7 @@ /// public string Create(string path, byte[] data, CreateMode mode) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.Client.Create(path, data, Ids.OPEN_ACL_UNSAFE, mode); @@ -193,7 +193,7 @@ /// public IList GetChildren(string path, bool watch) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.Client.GetChildren(path, watch); @@ -216,7 +216,7 @@ /// public byte[] ReadData(string path, Stat stats, bool watch) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); return this.Client.GetData(path, watch, stats); @@ -233,7 +233,7 @@ /// public void WriteData(string path, byte[] data) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); this.WriteData(path, data, -1); @@ -253,7 +253,7 @@ /// public void WriteData(string path, byte[] data, int version) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); this.Client.SetData(path, data, version); @@ -270,7 +270,7 @@ /// public long GetCreateTime(string path) { - Guard.Assert(() => !string.IsNullOrEmpty(path)); + Guard.NotNullNorEmpty(path, "path"); this.EnsuresNotDisposed(); Stat stats = this.Client.Exists(path, false); Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKSessionExpireListener.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKSessionExpireListener.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKSessionExpireListener.cs (working copy) @@ -58,7 +58,7 @@ /// public void HandleStateChanged(ZooKeeperStateChangedEventArgs args) { - Guard.Assert(() => args != null); + Guard.NotNull(args, "args"); Guard.Assert(() => args.State != KeeperState.Unknown); } @@ -72,7 +72,7 @@ /// public void HandleSessionCreated(ZooKeeperSessionCreatedEventArgs args) { - Guard.Assert(() => args != null); + Guard.NotNull(args, "args"); Logger.InfoFormat( CultureInfo.CurrentCulture, Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/BrokerTopicsListener.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/BrokerTopicsListener.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/BrokerTopicsListener.cs (working copy) @@ -77,9 +77,9 @@ /// public void HandleChildChange(ZooKeeperChildChangedEventArgs e) { - Guard.Assert(() => e != null); - Guard.Assert(() => !string.IsNullOrEmpty(e.Path)); - Guard.Assert(() => e.Children != null); + Guard.NotNull(e, "e"); + Guard.NotNullNorEmpty(e.Path, "e.Path"); + Guard.NotNull(e.Children, "e.Children"); lock (this.syncLock) { Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKRebalancerListener.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKRebalancerListener.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/Listeners/ZKRebalancerListener.cs (working copy) @@ -49,7 +49,7 @@ private readonly object syncLock; - private readonly ConsumerConfig config; + private readonly ConsumerConfiguration config; private readonly IZooKeeperClient zkClient; @@ -60,7 +60,7 @@ private readonly ZookeeperConsumerConnector zkConsumerConnector; internal ZKRebalancerListener( - ConsumerConfig config, + ConsumerConfiguration config, string consumerIdString, IDictionary> topicRegistry, IZooKeeperClient zkClient, @@ -106,7 +106,7 @@ //// release all partitions, reset state and retry this.ReleasePartitionOwnership(); this.ResetState(); - Thread.Sleep(config.ZkSyncTimeMs); + Thread.Sleep(config.ZooKeeper.ZkSyncTimeMs); } } @@ -124,9 +124,9 @@ /// public void HandleChildChange(ZooKeeperChildChangedEventArgs args) { - Guard.Assert(() => args != null); - Guard.Assert(() => !string.IsNullOrEmpty(args.Path)); - Guard.Assert(() => args.Children != null); + Guard.NotNull(args, "args"); + Guard.NotNullNorEmpty(args.Path, "args.Path"); + Guard.NotNull(args.Children, "args.Children"); SyncedRebalance(); } Index: clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperStringSerializer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperStringSerializer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/ZooKeeperIntegration/ZooKeeperStringSerializer.cs (working copy) @@ -48,7 +48,7 @@ /// public byte[] Serialize(object obj) { - Guard.Assert(() => obj != null); + Guard.NotNull(obj, "obj"); return Encoding.UTF8.GetBytes(obj.ToString()); } @@ -63,8 +63,8 @@ /// public object Deserialize(byte[] bytes) { - Guard.Assert(() => bytes != null); - Guard.Assert(() => bytes.Count() > 0); + Guard.NotNull(bytes, "bytes"); + Guard.Greater(bytes.Count(), 0, "bytes"); return bytes == null ? null : Encoding.UTF8.GetString(bytes); } Index: clients/csharp/src/Kafka/Kafka.Client/Messages/MessageSet.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Messages/MessageSet.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Messages/MessageSet.cs (working copy) @@ -48,7 +48,7 @@ /// public static int GetEntrySize(Message message) { - Guard.Assert(() => message != null); + Guard.NotNull(message, "message"); return message.Size + DefaultMessageLengthSize; } Index: clients/csharp/src/Kafka/Kafka.Client/Messages/BufferedMessageSet.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Messages/BufferedMessageSet.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Messages/BufferedMessageSet.cs (working copy) @@ -15,21 +15,39 @@ * limitations under the License. */ +using Kafka.Client.Exceptions; + namespace Kafka.Client.Messages { using System; + using System.Collections; using System.Collections.Generic; + using System.Globalization; using System.IO; using System.Linq; + using System.Reflection; using System.Text; + using Kafka.Client.Consumers; using Kafka.Client.Serialization; using Kafka.Client.Utils; + using log4net; /// /// A collection of messages stored as memory stream /// - public class BufferedMessageSet : MessageSet + public class BufferedMessageSet : MessageSet, IEnumerable, IEnumerator { + private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private MemoryStream topIter; + private int topIterPosition; + private long currValidBytes = 0; + private IEnumerator innerIter = null; + private long lastMessageSize = 0; + private long deepValidByteCount = -1; + private long shallowValidByteCount = -1; + private ConsumerIteratorState state = ConsumerIteratorState.NotReady; + private MessageAndOffset nextItem; + /// /// Gets the error code /// @@ -41,7 +59,8 @@ /// /// The list of messages. /// - public BufferedMessageSet(IEnumerable messages) : this(messages, ErrorMapping.NoError) + public BufferedMessageSet(IEnumerable messages) + : this(messages, ErrorMapping.NoError) { } @@ -58,16 +77,35 @@ { int length = GetMessageSetSize(messages); this.Messages = messages; - this.SetBuffer = new BoundedBuffer(length); - this.WriteTo(this.SetBuffer); this.ErrorCode = errorCode; + this.topIterPosition = 0; } /// - /// Gets set internal buffer + /// Initializes a new instance of the class with compression. /// - public MemoryStream SetBuffer { get; private set; } + /// compression method + /// messages to add + public BufferedMessageSet(CompressionCodecs compressionCodec, IEnumerable messages) + { + IEnumerable messagesToAdd; + switch (compressionCodec) + { + case CompressionCodecs.NoCompressionCodec: + messagesToAdd = messages; + break; + default: + var message = CompressionUtils.Compress(messages, compressionCodec); + messagesToAdd = new List() { message }; + break; + } + int length = GetMessageSetSize(messagesToAdd); + this.Messages = messagesToAdd; + this.ErrorCode = ErrorMapping.NoError; + this.topIterPosition = 0; + } + /// /// Gets the list of messages. /// @@ -78,12 +116,33 @@ /// public override int SetSize { - get + get { return GetMessageSetSize(this.Messages); } + } + + public MessageAndOffset Current + { + get { - return (int)this.SetBuffer.Length; + if (!MoveNext()) + { + throw new NoSuchElementException(); + } + + state = ConsumerIteratorState.NotReady; + if (nextItem != null) + { + return nextItem; + } + + throw new IllegalStateException("Expected item but none found."); } } + object IEnumerator.Current + { + get { return this.Current; } + } + /// /// Writes content into given stream /// @@ -92,7 +151,7 @@ /// public sealed override void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { this.WriteTo(writer); @@ -107,7 +166,7 @@ /// public sealed override void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); foreach (var message in this.Messages) { writer.Write(message.Size); @@ -123,38 +182,16 @@ /// public override string ToString() { - using (var reader = new KafkaBinaryReader(this.SetBuffer)) - { - return ParseFrom(reader, this.SetSize); - } - } - - /// - /// Helper method to get string representation of set - /// - /// - /// The reader. - /// - /// - /// The count. - /// - /// - /// String representation of set - /// - internal static string ParseFrom(KafkaBinaryReader reader, int count) - { - Guard.Assert(() => reader != null); var sb = new StringBuilder(); int i = 1; - while (reader.BaseStream.Position != reader.BaseStream.Length) + foreach (var message in this.Messages) { sb.Append("Message "); sb.Append(i); sb.Append(" {Length: "); - int msgSize = reader.ReadInt32(); - sb.Append(msgSize); + sb.Append(message.Size); sb.Append(", "); - sb.Append(Message.ParseFrom(reader, msgSize)); + sb.Append(message.ToString()); sb.AppendLine("} "); i++; } @@ -162,6 +199,77 @@ return sb.ToString(); } + internal static BufferedMessageSet ParseFrom(KafkaBinaryReader reader, int size) + { + if (size == 0) + { + return new BufferedMessageSet(Enumerable.Empty()); + } + + short errorCode = reader.ReadInt16(); + if (errorCode != KafkaException.NoError) + { + throw new KafkaException(errorCode); + } + + int readed = 2; + if (readed == size) + { + return new BufferedMessageSet(Enumerable.Empty()); + } + + var messages = new List(); + do + { + int msgSize = reader.ReadInt32(); + readed += 4; + Message msg = Message.ParseFrom(reader, msgSize); + readed += msgSize; + messages.Add(msg); + } + while (readed < size); + if (size != readed) + { + throw new KafkaException(KafkaException.InvalidRetchSizeCode); + } + + return new BufferedMessageSet(messages); + } + + internal static IList ParseMultiFrom(KafkaBinaryReader reader, int size, int count) + { + var result = new List(); + if (size == 0) + { + return result; + } + + int readed = 0; + short errorCode = reader.ReadInt16(); + readed += 2; + if (errorCode != KafkaException.NoError) + { + throw new KafkaException(errorCode); + } + + for (int i = 0; i < count; i++) + { + int partSize = reader.ReadInt32(); + readed += 4; + var item = ParseFrom(reader, partSize); + readed += partSize; + result.Add(item); + } + + if (size != readed) + { + throw new KafkaException(KafkaException.InvalidRetchSizeCode); + } + + return result; + } + + [Obsolete] internal static BufferedMessageSet ParseFrom(byte[] bytes) { var messages = new List(); @@ -176,5 +284,124 @@ return new BufferedMessageSet(messages); } + + public IEnumerator GetEnumerator() + { + return this; + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + private bool InnerDone() + { + return innerIter == null || !innerIter.MoveNext(); + } + + private MessageAndOffset MakeNextOuter() + { + if (topIterPosition >= this.Messages.Count()) + { + return AllDone(); + } + + Message newMessage = this.Messages.ToList()[topIterPosition]; + topIterPosition++; + switch (newMessage.CompressionCodec) + { + case CompressionCodecs.NoCompressionCodec: + if (Logger.IsDebugEnabled) + { + Logger.DebugFormat( + CultureInfo.CurrentCulture, + "Message is uncompressed. Valid byte count = {0}", + currValidBytes); + } + + innerIter = null; + currValidBytes += 4 + newMessage.Size; + return new MessageAndOffset(newMessage, currValidBytes); + default: + if (Logger.IsDebugEnabled) + { + Logger.DebugFormat(CultureInfo.CurrentCulture, "Message is compressed. Valid byte count = {0}", currValidBytes); + } + + innerIter = CompressionUtils.Decompress(newMessage).GetEnumerator(); + return MakeNext(); + } + } + + private MessageAndOffset MakeNext() + { + if (Logger.IsDebugEnabled) + { + Logger.DebugFormat(CultureInfo.CurrentCulture, "MakeNext() in deepIterator: innerDone = {0}", InnerDone()); + } + + switch (InnerDone()) + { + case true: + return MakeNextOuter(); + default: + var messageAndOffset = innerIter.Current; + if (!innerIter.MoveNext()) + { + currValidBytes += 4 + lastMessageSize; + } + + return new MessageAndOffset(messageAndOffset.Message, currValidBytes); + } + } + + private MessageAndOffset AllDone() + { + state = ConsumerIteratorState.Done; + return null; + } + + public void Dispose() + { + } + + public bool MoveNext() + { + if (state == ConsumerIteratorState.Failed) + { + throw new IllegalStateException("Iterator is in failed state"); + } + + switch (state) + { + case ConsumerIteratorState.Done: + return false; + case ConsumerIteratorState.Ready: + return true; + default: + return MaybeComputeNext(); + } + } + + private bool MaybeComputeNext() + { + state = ConsumerIteratorState.Failed; + nextItem = MakeNext(); + if (state == ConsumerIteratorState.Done) + { + return false; + } + else + { + state = ConsumerIteratorState.Ready; + return true; + } + } + + public void Reset() + { + this.topIterPosition = 0; + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Messages/Message.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Messages/Message.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Messages/Message.cs (working copy) @@ -21,6 +21,7 @@ using System.IO; using System.Linq; using System.Text; + using Kafka.Client.Exceptions; using Kafka.Client.Serialization; using Kafka.Client.Utils; @@ -35,11 +36,28 @@ /// public class Message : IWritable { - private const byte DefaultMagicValue = 0; + private const byte DefaultMagicValue = 1; private const byte DefaultMagicLength = 1; private const byte DefaultCrcLength = 4; private const int DefaultHeaderSize = DefaultMagicLength + DefaultCrcLength; + private const byte CompressionCodeMask = 3; + public CompressionCodecs CompressionCodec + { + get + { + switch (Magic) + { + case 0: + return CompressionCodecs.NoCompressionCodec; + case 1: + return Messages.CompressionCodec.GetCompressionCodec(Attributes & CompressionCodeMask); + default: + throw new KafkaException(KafkaException.InvalidMessageCode); + } + } + } + /// /// Initializes a new instance of the class. /// @@ -53,11 +71,11 @@ /// Initializes with default magic number /// public Message(byte[] payload, byte[] checksum) - : this(payload, DefaultMagicValue, checksum) + : this(payload, checksum, CompressionCodecs.NoCompressionCodec) { - Guard.Assert(() => payload != null); - Guard.Assert(() => checksum != null); - Guard.Assert(() => checksum.Length == 4); + Guard.NotNull(payload, "payload"); + Guard.NotNull(checksum, "checksum"); + Guard.Count(checksum, 4, "checksum"); } /// @@ -70,9 +88,9 @@ /// Initializes the magic number as default and the checksum as null. It will be automatically computed. /// public Message(byte[] payload) - : this(payload, DefaultMagicValue) + : this(payload, CompressionCodecs.NoCompressionCodec) { - Guard.Assert(() => payload != null); + Guard.NotNull(payload, "payload"); } /// @@ -83,10 +101,10 @@ /// /// The data for the payload. /// The magic identifier. - public Message(byte[] payload, byte magic) - : this(payload, magic, Crc32Hasher.Compute(payload)) + public Message(byte[] payload, CompressionCodecs compressionCodec) + : this(payload, Crc32Hasher.Compute(payload), compressionCodec) { - Guard.Assert(() => payload != null); + Guard.NotNull(payload, "payload"); } /// @@ -95,25 +113,32 @@ /// The data for the payload. /// The magic identifier. /// The checksum for the payload. - public Message(byte[] payload, byte magic, byte[] checksum) + public Message(byte[] payload, byte[] checksum, CompressionCodecs compressionCodec) { - Guard.Assert(() => payload != null); - Guard.Assert(() => checksum != null); + Guard.NotNull(payload, "payload"); + Guard.NotNull(checksum, "checksum"); + Guard.Count(checksum, 4, "checksum"); int length = DefaultHeaderSize + payload.Length; this.Payload = payload; - this.Magic = magic; + this.Magic = DefaultMagicValue; + + if (compressionCodec != CompressionCodecs.NoCompressionCodec) + { + this.Attributes |= + (byte)(CompressionCodeMask & Messages.CompressionCodec.GetCompressionCodecValue(compressionCodec)); + } + + if (Magic == 1) + { + length++; + } + this.Checksum = checksum; - this.MessageBuffer = new BoundedBuffer(length); - this.WriteTo(this.MessageBuffer); + this.Size = length; } /// - /// Gets internal message buffer. - /// - public MemoryStream MessageBuffer { get; private set; } - - /// /// Gets the payload. /// public byte[] Payload { get; private set; } @@ -129,15 +154,14 @@ public byte[] Checksum { get; private set; } /// + /// Gets the Attributes for the message. + /// + public byte Attributes { get; private set; } + + /// /// Gets the total size of message. /// - public int Size - { - get - { - return (int)this.MessageBuffer.Length; - } - } + public int Size { get; private set; } /// /// Gets the payload size. @@ -158,7 +182,7 @@ /// public void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { @@ -174,9 +198,9 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); - + Guard.NotNull(writer, "writer"); writer.Write(this.Magic); + writer.Write(this.Attributes); writer.Write(this.Checksum); writer.Write(this.Payload); } @@ -187,44 +211,27 @@ /// The decoded payload as string. public override string ToString() { - using (var reader = new KafkaBinaryReader(this.MessageBuffer)) + var sb = new StringBuilder(); + sb.Append("Magic: "); + sb.Append(this.Magic); + if (this.Magic == 1) { - return ParseFrom(reader, this.Size); + sb.Append(", Attributes: "); + sb.Append(this.Attributes); } - } - /// - /// Creates string representation of message - /// - /// - /// The reader. - /// - /// - /// The count. - /// - /// - /// String representation of message - /// - public static string ParseFrom(KafkaBinaryReader reader, int count) - { - Guard.Assert(() => reader != null); - var sb = new StringBuilder(); - int payloadSize = count - DefaultHeaderSize; - sb.Append("Magic: "); - sb.Append(reader.ReadByte()); sb.Append(", Checksum: "); for (int i = 0; i < 4; i++) { sb.Append("["); - sb.Append(reader.ReadByte()); + sb.Append(this.Checksum[i]); sb.Append("]"); } sb.Append(", topic: "); - var encodedPayload = reader.ReadBytes(payloadSize); try { - sb.Append(Encoding.UTF8.GetString(encodedPayload)); + sb.Append(Encoding.UTF8.GetString(this.Payload)); } catch (Exception) { @@ -234,6 +241,63 @@ return sb.ToString(); } + [Obsolete("Use KafkaBinaryReader instead")] + public static Message FromMessageBytes(byte[] data) + { + byte magic = data[0]; + byte[] checksum; + byte[] payload; + byte attributes; + if (magic == (byte)1) + { + attributes = data[1]; + checksum = data.Skip(2).Take(4).ToArray(); + payload = data.Skip(6).ToArray(); + return new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)); + } + else + { + checksum = data.Skip(1).Take(4).ToArray(); + payload = data.Skip(5).ToArray(); + return new Message(payload, checksum); + } + } + + internal static Message ParseFrom(KafkaBinaryReader reader, int size) + { + Message result; + int readed = 0; + byte magic = reader.ReadByte(); + readed++; + byte[] checksum; + byte[] payload; + if (magic == 1) + { + byte attributes = reader.ReadByte(); + readed++; + checksum = reader.ReadBytes(4); + readed += 4; + payload = reader.ReadBytes(size - (DefaultHeaderSize + 1)); + readed += size - (DefaultHeaderSize + 1); + result = new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)); + } + else + { + checksum = reader.ReadBytes(4); + readed += 4; + payload = reader.ReadBytes(size - DefaultHeaderSize); + readed += size - DefaultHeaderSize; + result = new Message(payload, checksum); + } + + if (size != readed) + { + throw new KafkaException(KafkaException.InvalidRetchSizeCode); + } + + return result; + } + /// /// Parses a message from a byte array given the format Kafka likes. /// @@ -244,10 +308,22 @@ { int size = BitConverter.ToInt32(BitWorks.ReverseBytes(data.Take(4).ToArray()), 0); byte magic = data[4]; - byte[] checksum = data.Skip(5).Take(4).ToArray(); - byte[] payload = data.Skip(9).Take(size).ToArray(); - - return new Message(payload, magic, checksum); + byte[] checksum; + byte[] payload; + byte attributes; + if (magic == 1) + { + attributes = data[5]; + checksum = data.Skip(6).Take(4).ToArray(); + payload = data.Skip(10).Take(size).ToArray(); + return new Message(payload, checksum, Messages.CompressionCodec.GetCompressionCodec(attributes & CompressionCodeMask)); + } + else + { + checksum = data.Skip(5).Take(4).ToArray(); + payload = data.Skip(9).Take(size).ToArray(); + return new Message(payload, checksum); + } } } } Index: clients/csharp/src/Kafka/Kafka.Client/Requests/ProducerRequest.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Requests/ProducerRequest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Requests/ProducerRequest.cs (working copy) @@ -35,7 +35,6 @@ public const byte DefaultPartitionSize = 4; public const byte DefaultSetSizeSize = 4; public const byte DefaultHeaderSize = DefaultRequestSizeSize + DefaultTopicSizeSize + DefaultPartitionSize + DefaultRequestIdSize + DefaultSetSizeSize; - public const short DefaultTopicLengthIfNonePresent = 2; public static int GetRequestLength(string topic, int messegesSize, string encoding = DefaultEncoding) { @@ -45,7 +44,8 @@ public ProducerRequest(string topic, int partition, BufferedMessageSet messages) { - Guard.Assert(() => messages != null); + Guard.NotNull(messages, "messages"); + int length = GetRequestLength(topic, messages.SetSize); this.RequestBuffer = new BoundedBuffer(length); this.Topic = topic; @@ -91,7 +91,8 @@ /// public void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); + using (var writer = new KafkaBinaryWriter(output)) { writer.Write(this.RequestBuffer.Capacity - DefaultRequestSizeSize); @@ -108,7 +109,8 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); + writer.WriteTopic(this.Topic, DefaultEncoding); writer.Write(this.Partition); writer.Write(this.MessageSet.SetSize); @@ -117,39 +119,22 @@ public override string ToString() { - using (var reader = new KafkaBinaryReader(this.RequestBuffer)) - { - return ParseFrom(reader, this.TotalSize); - } - } - - public static string ParseFrom(KafkaBinaryReader reader, int count, bool skipReqInfo = false) - { - Guard.Assert(() => reader != null); var sb = new StringBuilder(); - - if (!skipReqInfo) - { - sb.Append("Request size: "); - sb.Append(reader.ReadInt32()); - sb.Append(", RequestId: "); - short reqId = reader.ReadInt16(); - sb.Append(reqId); - sb.Append("("); - sb.Append((RequestTypes)reqId); - sb.Append(")"); - } - + sb.Append("Request size: "); + sb.Append(this.TotalSize); + sb.Append(", RequestId: "); + sb.Append(this.RequestTypeId); + sb.Append("("); + sb.Append((RequestTypes)this.RequestTypeId); + sb.Append(")"); sb.Append(", Topic: "); - string topic = reader.ReadTopic(DefaultEncoding); - sb.Append(topic); + sb.Append(this.Topic); sb.Append(", Partition: "); - sb.Append(reader.ReadInt32()); + sb.Append(this.Partition); sb.Append(", Set size: "); - sb.Append(reader.ReadInt32()); - int size = count - DefaultHeaderSize - GetTopicLength(topic); + sb.Append(this.MessageSet.SetSize); sb.Append(", Set {"); - sb.Append(BufferedMessageSet.ParseFrom(reader, size)); + sb.Append(this.MessageSet.ToString()); sb.Append("}"); return sb.ToString(); } Index: clients/csharp/src/Kafka/Kafka.Client/Requests/MultiFetchRequest.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Requests/MultiFetchRequest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Requests/MultiFetchRequest.cs (working copy) @@ -51,7 +51,7 @@ /// Requests to package up and batch. public MultiFetchRequest(IList requests) { - Guard.Assert(() => requests != null); + Guard.NotNull(requests, "requests"); ConsumerRequests = requests; int length = GetRequestLength(requests, DefaultEncoding); this.RequestBuffer = new BoundedBuffer(length); @@ -79,7 +79,7 @@ /// public void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { @@ -98,7 +98,7 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); foreach (var consumerRequest in ConsumerRequests) { Index: clients/csharp/src/Kafka/Kafka.Client/Requests/OffsetRequest.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Requests/OffsetRequest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Requests/OffsetRequest.cs (working copy) @@ -107,7 +107,7 @@ /// public void WriteTo(System.IO.MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { @@ -125,7 +125,7 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); writer.WriteTopic(this.Topic, DefaultEncoding); writer.Write(this.Partition); Index: clients/csharp/src/Kafka/Kafka.Client/Requests/FetchRequest.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Requests/FetchRequest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Requests/FetchRequest.cs (working copy) @@ -116,7 +116,7 @@ /// public void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { @@ -134,7 +134,7 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); writer.WriteTopic(this.Topic, DefaultEncoding); writer.Write(this.Partition); Index: clients/csharp/src/Kafka/Kafka.Client/Requests/MultiProducerRequest.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Requests/MultiProducerRequest.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Requests/MultiProducerRequest.cs (working copy) @@ -35,7 +35,7 @@ public static int GetBufferLength(IEnumerable requests) { - Guard.Assert(() => requests != null); + Guard.NotNull(requests, "requests"); return DefaultRequestSizeSize + DefaultRequestIdSize @@ -51,7 +51,8 @@ /// public MultiProducerRequest(IEnumerable requests) { - Guard.Assert(() => requests != null); + Guard.NotNull(requests, "requests"); + int length = GetBufferLength(requests); ProducerRequests = requests; this.RequestBuffer = new BoundedBuffer(length); @@ -79,7 +80,7 @@ /// public void WriteTo(MemoryStream output) { - Guard.Assert(() => output != null); + Guard.NotNull(output, "output"); using (var writer = new KafkaBinaryWriter(output)) { @@ -97,7 +98,7 @@ /// public void WriteTo(KafkaBinaryWriter writer) { - Guard.Assert(() => writer != null); + Guard.NotNull(writer, "writer"); writer.Write((short)this.ProducerRequests.Count()); foreach (var request in ProducerRequests) @@ -108,33 +109,21 @@ public override string ToString() { - using (var reader = new KafkaBinaryReader(this.RequestBuffer)) - { - return ParseFrom(reader, (int)this.RequestBuffer.Length); - } - } - - public static string ParseFrom(KafkaBinaryReader reader, int count) - { - Guard.Assert(() => reader != null); - var sb = new StringBuilder(); sb.Append("Request size: "); - sb.Append(reader.ReadInt32()); + sb.Append(this.RequestBuffer.Capacity - DefaultRequestSizeSize); sb.Append(", RequestId: "); - short reqId = reader.ReadInt16(); - sb.Append(reqId); + sb.Append(this.RequestTypeId); sb.Append("("); - sb.Append((RequestTypes)reqId); + sb.Append((RequestTypes)this.RequestTypeId); sb.Append("), Single Requests: {"); int i = 1; - while (reader.BaseStream.Position != reader.BaseStream.Length) + foreach (var request in ProducerRequests) { sb.Append("Request "); sb.Append(i); sb.Append(" {"); - int msgSize = 0; - sb.Append(ProducerRequest.ParseFrom(reader, msgSize)); + sb.Append(request.ToString()); sb.AppendLine("} "); i++; } Index: clients/csharp/src/Kafka/Kafka.Client/Serialization/KafkaBinaryReader.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Serialization/KafkaBinaryReader.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Serialization/KafkaBinaryReader.cs (working copy) @@ -20,6 +20,7 @@ using System.IO; using System.Net; using System.Text; + using System.Net.Sockets; /// /// Reads data from underlying stream using big endian bytes order for primitive types @@ -47,7 +48,10 @@ /// protected override void Dispose(bool disposing) { - this.BaseStream.Position = 0; + if (this.BaseStream.CanSeek) + { + this.BaseStream.Position = 0; + } } /// @@ -127,5 +131,18 @@ Encoding encoder = Encoding.GetEncoding(encoding); return encoder.GetString(bytes); } + + public bool DataAvailabe + { + get + { + if (this.BaseStream is NetworkStream) + { + return ((NetworkStream)this.BaseStream).DataAvailable; + } + + return this.BaseStream.Length != this.BaseStream.Position; + } + } } } Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/Consumer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/Consumer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/Consumer.cs (working copy) @@ -20,7 +20,6 @@ using System; using System.Collections.Generic; using System.Globalization; - using System.Linq; using System.Reflection; using Kafka.Client.Cfg; using Kafka.Client.Exceptions; @@ -33,25 +32,32 @@ /// The low-level API of consumer of Kafka messages /// /// - /// Maintains a connection to a single broker and has a close correspondence + /// Maintains a connection to a single broker and has a close correspondence /// to the network requests sent to the server. /// Also, is completely stateless. /// public class Consumer : IConsumer { - private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private readonly ConsumerConfig config; + private readonly ConsumerConfiguration config; + private readonly string host; + private readonly int port; /// - /// Gets the server to which the connection is to be established. + /// Initializes a new instance of the class. /// - public string Host { get; private set; } + /// + /// The consumer configuration. + /// + public Consumer(ConsumerConfiguration config) + { + Guard.NotNull(config, "config"); - /// - /// Gets the port to which the connection is to be established. - /// - public int Port { get; private set; } + this.config = config; + this.host = config.Broker.Host; + this.port = config.Broker.Port; + } /// /// Initializes a new instance of the class. @@ -59,13 +65,15 @@ /// /// The consumer configuration. /// - public Consumer(ConsumerConfig config) + /// + /// + public Consumer(ConsumerConfiguration config, string host, int port) { - Guard.Assert(() => config != null); + Guard.NotNull(config, "config"); this.config = config; - this.Host = config.Host; - this.Port = config.Port; + this.host = host; + this.port = port; } /// @@ -78,38 +86,41 @@ /// A set of fetched messages. /// /// - /// Offset is passed in on every request, allowing the user to maintain this metadata + /// Offset is passed in on every request, allowing the user to maintain this metadata /// however they choose. /// public BufferedMessageSet Fetch(FetchRequest request) { - BufferedMessageSet result = null; - using (var conn = new KafkaConnection(this.Host, this.Port)) + short tryCounter = 1; + while (tryCounter <= this.config.NumberOfTries) { - short tryCounter = 1; - bool success = false; - while (!success && tryCounter <= this.config.NumberOfTries) + try { - try + using (var conn = new KafkaConnection( + this.host, + this.port, + this.config.BufferSize, + this.config.SocketTimeout)) { - result = Fetch(conn, request); - success = true; + conn.Write(request); + int size = conn.Reader.ReadInt32(); + return BufferedMessageSet.ParseFrom(conn.Reader, size); } - catch (Exception ex) + } + catch (Exception ex) + { + //// if maximum number of tries reached + if (tryCounter == this.config.NumberOfTries) { - //// if maximum number of tries reached - if (tryCounter == this.config.NumberOfTries) - { - throw; - } - - tryCounter++; - Logger.InfoFormat(CultureInfo.CurrentCulture, "Fetch reconnect due to {0}", ex); + throw; } + + tryCounter++; + Logger.InfoFormat(CultureInfo.CurrentCulture, "Fetch reconnect due to {0}", ex); } } - return result; + return null; } /// @@ -128,28 +139,32 @@ public IList MultiFetch(MultiFetchRequest request) { var result = new List(); - using (var conn = new KafkaConnection(this.Host, this.Port)) + short tryCounter = 1; + while (tryCounter <= this.config.NumberOfTries) { - short tryCounter = 1; - bool success = false; - while (!success && tryCounter <= this.config.NumberOfTries) + try { - try + using (var conn = new KafkaConnection( + this.host, + this.port, + this.config.BufferSize, + this.config.SocketTimeout)) { - MultiFetch(conn, request, result); - success = true; + conn.Write(request); + int size = conn.Reader.ReadInt32(); + return BufferedMessageSet.ParseMultiFrom(conn.Reader, size, request.ConsumerRequests.Count); } - catch (Exception ex) + } + catch (Exception ex) + { + // if maximum number of tries reached + if (tryCounter == this.config.NumberOfTries) { - // if maximum number of tries reached - if (tryCounter == this.config.NumberOfTries) - { - throw; - } - - tryCounter++; - Logger.InfoFormat(CultureInfo.CurrentCulture, "MultiFetch reconnect due to {0}", ex); + throw; } + + tryCounter++; + Logger.InfoFormat(CultureInfo.CurrentCulture, "MultiFetch reconnect due to {0}", ex); } } @@ -167,122 +182,54 @@ /// public IList GetOffsetsBefore(OffsetRequest request) { - var offsets = new List(); - using (var conn = new KafkaConnection(this.Host, this.Port)) + var result = new List(); + short tryCounter = 1; + while (tryCounter <= this.config.NumberOfTries) { - short tryCounter = 1; - bool success = false; - while (!success && tryCounter <= this.config.NumberOfTries) + try { - try + using (var conn = new KafkaConnection( + this.host, + this.port, + this.config.BufferSize, + this.config.SocketTimeout)) { - GetOffsetsBefore(conn, request, offsets); - success = true; - } - catch (Exception ex) - { - // if maximum number of tries reached - if (tryCounter == this.config.NumberOfTries) + conn.Write(request); + int size = conn.Reader.ReadInt32(); + if (size == 0) { - throw; + return result; } - tryCounter++; - Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex); - } - } - } + short errorCode = conn.Reader.ReadInt16(); + if (errorCode != KafkaException.NoError) + { + throw new KafkaException(errorCode); + } - return offsets; - } + int count = conn.Reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + result.Add(conn.Reader.ReadInt64()); + } - private static BufferedMessageSet Fetch(KafkaConnection conn, FetchRequest request) - { - conn.Write(request); - int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0); - if (dataLength > 0) - { - byte[] data = conn.Read(dataLength); - - int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0); - if (errorCode != KafkaException.NoError) - { - throw new KafkaException(errorCode); + return result; + } } - - // skip the error code - byte[] unbufferedData = data.Skip(2).ToArray(); - return BufferedMessageSet.ParseFrom(unbufferedData); - } - - return null; - } - - private static void MultiFetch(KafkaConnection conn, MultiFetchRequest request, IList result) - { - result.Clear(); - conn.Write(request); - int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0); - if (dataLength <= 0) - { - return; - } - - byte[] data = conn.Read(dataLength); - - int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0); - if (errorCode != KafkaException.NoError) - { - throw new KafkaException(errorCode); - } - - // skip the error code - byte[] unbufferedData = data.Skip(2).ToArray(); - for (int i = 0; i < request.ConsumerRequests.Count; i++) - { - int partLength = BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0); - errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(unbufferedData.Skip(4).Take(2).ToArray()), 0); - if (errorCode != KafkaException.NoError) + catch (Exception ex) { - throw new KafkaException(errorCode); - } + //// if maximum number of tries reached + if (tryCounter == this.config.NumberOfTries) + { + throw; + } - result.Add(BufferedMessageSet.ParseFrom(unbufferedData.Skip(6).Take(partLength - 2).ToArray())); - unbufferedData = unbufferedData.Skip(partLength + 4).ToArray(); - } - } - - private static void GetOffsetsBefore(KafkaConnection conn, OffsetRequest request, IList offsets) - { - offsets.Clear(); // to make sure the list is clean after some previous attampts to get data - conn.Write(request); - int dataLength = BitConverter.ToInt32(BitWorks.ReverseBytes(conn.Read(4)), 0); - - if (dataLength > 0) - { - byte[] data = conn.Read(dataLength); - - int errorCode = BitConverter.ToInt16(BitWorks.ReverseBytes(data.Take(2).ToArray()), 0); - if (errorCode != KafkaException.NoError) - { - throw new KafkaException(errorCode); + tryCounter++; + Logger.InfoFormat(CultureInfo.CurrentCulture, "GetOffsetsBefore reconnect due to {0}", ex); } - - // skip the error code and process the rest - byte[] unbufferedData = data.Skip(2).ToArray(); - - // first four bytes are the number of offsets - int numOfOffsets = - BitConverter.ToInt32(BitWorks.ReverseBytes(unbufferedData.Take(4).ToArray()), 0); - - for (int ix = 0; ix < numOfOffsets; ix++) - { - int position = (ix * 8) + 4; - offsets.Add( - BitConverter.ToInt64( - BitWorks.ReverseBytes(unbufferedData.Skip(position).Take(8).ToArray()), 0)); - } } + + return result; } } } Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/Fetcher.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/Fetcher.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/Fetcher.cs (working copy) @@ -33,7 +33,7 @@ internal class Fetcher : IDisposable { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private readonly ConsumerConfig config; + private readonly ConsumerConfiguration config; private readonly IZooKeeperClient zkClient; private FetcherRunnable[] fetcherWorkerObjects; private volatile bool disposed; @@ -48,7 +48,7 @@ /// /// The wrapper above ZooKeeper client. /// - public Fetcher(ConsumerConfig config, IZooKeeperClient zkClient) + public Fetcher(ConsumerConfiguration config, IZooKeeperClient zkClient) { this.config = config; this.zkClient = zkClient; Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/IConsumer.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/IConsumer.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/IConsumer.cs (working copy) @@ -31,16 +31,6 @@ public interface IConsumer { /// - /// Gets the server to which the connection is to be established. - /// - string Host { get; } - - /// - /// Gets the port to which the connection is to be established. - /// - int Port { get; } - - /// /// Fetch a set of messages from a topic. /// /// Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/ConsumerIterator.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/ConsumerIterator.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/ConsumerIterator.cs (working copy) @@ -40,8 +40,10 @@ private readonly int consumerTimeoutMs; private PartitionTopicInfo currentTopicInfo; private ConsumerIteratorState state = ConsumerIteratorState.NotReady; - private IEnumerator current; + private IEnumerator current; + private FetchedDataChunk currentDataChunk = null; private Message nextItem; + private long consumedOffset = -1; /// /// Initializes a new instance of the class. @@ -70,17 +72,27 @@ { if (!MoveNext()) { - throw new Exception("No element"); + throw new NoSuchElementException(); } state = ConsumerIteratorState.NotReady; if (nextItem != null) { - currentTopicInfo.Consumed(MessageSet.GetEntrySize(nextItem)); + if (consumedOffset < 0) + { + throw new IllegalStateException(String.Format(CultureInfo.CurrentCulture, "Offset returned by the message set is invalid {0}.", consumedOffset)); + } + + currentTopicInfo.ResetConsumeOffset(consumedOffset); + if (Logger.IsDebugEnabled) + { + Logger.DebugFormat(CultureInfo.CurrentCulture, "Setting consumed offset to {0}", consumedOffset); + } + return nextItem; } - throw new Exception("Expected item but none found."); + throw new IllegalStateException("Expected item but none found."); } } @@ -105,7 +117,7 @@ { if (state == ConsumerIteratorState.Failed) { - throw new Exception("Iterator is in failed state"); + throw new IllegalStateException("Iterator is in failed state"); } switch (state) @@ -148,14 +160,13 @@ { if (current == null || !current.MoveNext()) { - FetchedDataChunk found; if (consumerTimeoutMs < 0) { - found = this.channel.Take(); + currentDataChunk = this.channel.Take(); } else { - bool done = channel.TryTake(out found, consumerTimeoutMs); + bool done = channel.TryTake(out currentDataChunk, consumerTimeoutMs); if (!done) { Logger.Debug("Consumer iterator timing out..."); @@ -163,30 +174,32 @@ } } - if (found.Equals(ZookeeperConsumerConnector.ShutdownCommand)) + if (currentDataChunk.Equals(ZookeeperConsumerConnector.ShutdownCommand)) { Logger.Debug("Received the shutdown command"); - channel.Add(found); + channel.Add(currentDataChunk); return this.AllDone(); } - currentTopicInfo = found.TopicInfo; - if (currentTopicInfo.GetConsumeOffset() != found.FetchOffset) + currentTopicInfo = currentDataChunk.TopicInfo; + if (currentTopicInfo.GetConsumeOffset() != currentDataChunk.FetchOffset) { Logger.ErrorFormat( CultureInfo.CurrentCulture, "consumed offset: {0} doesn't match fetch offset: {1} for {2}; consumer may lose data", currentTopicInfo.GetConsumeOffset(), - found.FetchOffset, + currentDataChunk.FetchOffset, currentTopicInfo); - currentTopicInfo.ResetConsumeOffset(found.FetchOffset); + currentTopicInfo.ResetConsumeOffset(currentDataChunk.FetchOffset); } - current = found.Messages.Messages.GetEnumerator(); + current = currentDataChunk.Messages.GetEnumerator(); current.MoveNext(); } - return current.Current; + var item = current.Current; + consumedOffset = item.Offset; + return item.Message; } private Message AllDone() Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/FetcherRunnable.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/FetcherRunnable.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/FetcherRunnable.cs (working copy) @@ -42,7 +42,7 @@ private readonly IZooKeeperClient zkClient; - private readonly ConsumerConfig config; + private readonly ConsumerConfiguration config; private readonly Broker broker; @@ -52,14 +52,15 @@ private bool shouldStop; - internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfig config, Broker broker, List partitionTopicInfos) + internal FetcherRunnable(string name, IZooKeeperClient zkClient, ConsumerConfiguration config, Broker broker, List partitionTopicInfos) { this.name = name; this.zkClient = zkClient; this.config = config; this.broker = broker; this.partitionTopicInfos = partitionTopicInfos; - this.simpleConsumer = new Consumer(this.config); + + this.simpleConsumer = new Consumer(this.config, broker.Host, broker.Port); } /// @@ -87,7 +88,7 @@ var requestList = new List(); foreach (var partitionTopicInfo in this.partitionTopicInfos) { - var singleRequest = new FetchRequest(partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.config.FetchSize); + var singleRequest = new FetchRequest(partitionTopicInfo.Topic, partitionTopicInfo.Partition.PartId, partitionTopicInfo.GetFetchOffset(), this.config.MaxFetchSize); requestList.Add(singleRequest); } @@ -138,8 +139,8 @@ Logger.Info("Fetched bytes: " + read); if (read == 0) { - Logger.DebugFormat(CultureInfo.CurrentCulture, "backing off {0} ms", this.config.BackOffIncrementMs); - Thread.Sleep(this.config.BackOffIncrementMs); + Logger.DebugFormat(CultureInfo.CurrentCulture, "backing off {0} ms", this.config.BackOffIncrement); + Thread.Sleep(this.config.BackOffIncrement); } } } Index: clients/csharp/src/Kafka/Kafka.Client/Consumers/ZookeeperConsumerConnector.cs =================================================================== --- clients/csharp/src/Kafka/Kafka.Client/Consumers/ZookeeperConsumerConnector.cs (revision 1184850) +++ clients/csharp/src/Kafka/Kafka.Client/Consumers/ZookeeperConsumerConnector.cs (working copy) @@ -33,7 +33,7 @@ /// The consumer high-level API, that hides the details of brokers from the consumer. /// It also maintains the state of what has been consumed. /// - public class ZookeeperConsumerConnector : ZooKeeperAwareKafkaClientBase, IConsumerConnector + public class ZookeeperConsumerConnector : KafkaClientBase, IConsumerConnector { private static readonly ILog Logger = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); @@ -41,7 +41,7 @@ internal static readonly FetchedDataChunk ShutdownCommand = new FetchedDataChunk(null, null, -1); - private readonly ConsumerConfig config; + private readonly ConsumerConfiguration config; private IZooKeeperClient zkClient; @@ -79,8 +79,7 @@ /// /// Indicates whether fetchers should be enabled /// - public ZookeeperConsumerConnector(ConsumerConfig config, bool enableFetcher) - : base(config) + public ZookeeperConsumerConnector(ConsumerConfiguration config, bool enableFetcher) { this.config = config; this.enableFetcher = enableFetcher; @@ -89,8 +88,8 @@ if (this.config.AutoCommit) { - Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitIntervalMs); - scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitIntervalMs, this.config.AutoCommitIntervalMs); + Logger.InfoFormat(CultureInfo.CurrentCulture, "starting auto committer every {0} ms", this.config.AutoCommitInterval); + scheduler.ScheduleWithRate(this.AutoCommit, this.config.AutoCommitInterval, this.config.AutoCommitInterval); } } @@ -210,8 +209,8 @@ private void ConnectZk() { - Logger.InfoFormat(CultureInfo.CurrentCulture, "Connecting to zookeeper instance at {0}", this.config.ZkConnect); - this.zkClient = new ZooKeeperClient(this.config.ZkConnect, this.config.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); + Logger.InfoFormat(CultureInfo.CurrentCulture, "Connecting to zookeeper instance at {0}", this.config.ZooKeeper.ZkConnect); + this.zkClient = new ZooKeeperClient(this.config.ZooKeeper.ZkConnect, this.config.ZooKeeper.ZkSessionTimeoutMs, ZooKeeperStringSerializer.Serializer); this.zkClient.Connect(); }