Index: system_test/mirror_maker_testsuite/mirror_maker_test.py =================================================================== --- system_test/mirror_maker_testsuite/mirror_maker_test.py (revision 1397151) +++ system_test/mirror_maker_testsuite/mirror_maker_test.py (working copy) @@ -170,9 +170,6 @@ self.anonLogger.info("sleeping for 5s") time.sleep(5) - #print "#### sleeping for 30 min" - #time.sleep(1800) - self.log_message("creating topics") kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) self.anonLogger.info("sleeping for 5s") @@ -189,7 +186,7 @@ time.sleep(int(msgProducingFreeTimeSec)) # ============================================= - # A while-loop to bounce leader as specified + # A while-loop to bounce mirror maker as specified # by "num_iterations" in testcase_n_properties.json # ============================================= i = 1 @@ -198,47 +195,26 @@ self.log_message("Iteration " + str(i) + " of " + str(numIterations)) - self.log_message("looking up leader") - leaderDict = kafka_system_test_utils.get_leader_elected_log_line( - self.systemTestEnv, self.testcaseEnv, self.leaderAttributesDict) - - # ========================== - # leaderDict looks like this: - # ========================== - #{'entity_id': u'3', - # 'partition': '0', - # 'timestamp': 1345050255.8280001, - # 'hostname': u'localhost', - # 'topic': 'test_1', - # 'brokerid': '3'} - # ============================================= - # validate to see if leader election is successful + # Bounce Mirror Maker if specified in testcase config # ============================================= - self.log_message("validating leader election") - result = kafka_system_test_utils.validate_leader_election_successful( - self.testcaseEnv, leaderDict, self.testcaseEnv.validationStatusDict) - - # ============================================= - # trigger leader re-election by stopping leader - # to get re-election latency - # ============================================= - bounceLeaderFlag = self.testcaseEnv.testcaseArgumentsDict["bounce_leader"] - self.log_message("bounce_leader flag : " + bounceLeaderFlag) - if (bounceLeaderFlag.lower() == "true"): - reelectionLatency = kafka_system_test_utils.get_reelection_latency( - self.systemTestEnv, self.testcaseEnv, leaderDict, self.leaderAttributesDict) - latencyKeyName = "Leader Election Latency - iter " + str(i) + " brokerid " + leaderDict["brokerid"] - self.testcaseEnv.validationStatusDict[latencyKeyName] = str("{0:.2f}".format(reelectionLatency * 1000)) + " ms" - - # ============================================= - # starting previously terminated broker - # ============================================= - if bounceLeaderFlag.lower() == "true": - self.log_message("starting the previously terminated broker") - stoppedLeaderEntityId = leaderDict["entity_id"] - kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedLeaderEntityId) + bounceMirrorMaker = self.testcaseEnv.testcaseArgumentsDict["bounce_mirror_maker"] + self.log_message("bounce_mirror_maker flag : " + bounceMirrorMaker) + if (bounceMirrorMaker.lower() == "true"): + clusterConfigList = self.systemTestEnv.clusterEntityConfigDictList + mirrorMakerEntityIdList = system_test_utils.get_data_from_list_of_dicts( + clusterConfigList, "role", "mirror_maker", "entity_id") + + mirrorMakerPPid = self.testcaseEnv.entityMirrorMakerParentPidDict[mirrorMakerEntityIdList[0]] + self.log_message("stopping mirror maker : " + mirrorMakerPPid) + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, mirrorMakerEntityIdList[0], mirrorMakerPPid) + time.sleep(1) + + # starting previously terminated broker + self.log_message("starting the previously terminated mirror maker") + kafka_system_test_utils.start_mirror_makers(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 15s") time.sleep(15) i += 1 Index: system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json =================================================================== --- system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json (revision 1397151) +++ system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json (working copy) @@ -11,6 +11,7 @@ }, "testcase_args": { "bounce_leader": "false", + "bounce_mirror_maker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json =================================================================== --- system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json (revision 0) +++ system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json (revision 0) @@ -0,0 +1,136 @@ +{ + "description": {"01":"Replication with Mirror Maker => Bounce Mirror Maker", + "02":"Set up 2 clusters such as : SOURCE => MirrorMaker => TARGET", + "03":"Set up 2-node Zk cluster for both SOURCE & TARGET", + "04":"Produce and consume messages to a single topic - single partition.", + "05":"This test sends messages to 3 replicas", + "06":"At the end it verifies the log size and contents", + "07":"Use a consumer to verify no message loss in TARGET cluster.", + "08":"Producer dimensions : mode:sync, acks:-1, comp:0", + "09":"Log segment size : 10240" + }, + "testcase_args": { + "bounce_leader": "false", + "bounce_mirror_maker": "true", + "replica_factor": "3", + "num_partition": "1", + "num_iteration": "1", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2108", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_0.log", + "config_filename": "zookeeper_0.properties" + }, + { + "entity_id": "1", + "clientPort": "2118", + "dataDir": "/tmp/zookeeper_1", + "log_filename": "zookeeper_1.log", + "config_filename": "zookeeper_1.properties" + }, + + { + "entity_id": "2", + "clientPort": "2128", + "dataDir": "/tmp/zookeeper_2", + "log_filename": "zookeeper_2.log", + "config_filename": "zookeeper_2.properties" + }, + { + "entity_id": "3", + "clientPort": "2138", + "dataDir": "/tmp/zookeeper_3", + "log_filename": "zookeeper_3.log", + "config_filename": "zookeeper_3.properties" + }, + + { + "entity_id": "4", + "port": "9091", + "brokerid": "1", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_4_logs", + "log_filename": "kafka_server_4.log", + "config_filename": "kafka_server_4.properties" + }, + { + "entity_id": "5", + "port": "9092", + "brokerid": "2", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_5_logs", + "log_filename": "kafka_server_5.log", + "config_filename": "kafka_server_5.properties" + }, + { + "entity_id": "6", + "port": "9093", + "brokerid": "3", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_6_logs", + "log_filename": "kafka_server_6.log", + "config_filename": "kafka_server_6.properties" + }, + { + "entity_id": "7", + "port": "9094", + "brokerid": "4", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_7_logs", + "log_filename": "kafka_server_7.log", + "config_filename": "kafka_server_7.properties" + }, + { + "entity_id": "8", + "port": "9095", + "brokerid": "5", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_8_logs", + "log_filename": "kafka_server_8.log", + "config_filename": "kafka_server_8.properties" + }, + { + "entity_id": "9", + "port": "9096", + "brokerid": "6", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_9_logs", + "log_filename": "kafka_server_9.log", + "config_filename": "kafka_server_9.properties" + }, + + { + "entity_id": "10", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance_10.log", + "config_filename": "producer_performance_10.properties" + }, + { + "entity_id": "11", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "log_filename": "console_consumer_11.log", + "config_filename": "console_consumer_11.properties" + }, + + { + "entity_id": "12", + "log_filename": "mirror_maker_12.log", + "mirror_consumer_config_filename": "mirror_consumer_12.properties", + "mirror_producer_config_filename": "mirror_producer_12.properties" + } + ] +} Index: system_test/testcase_to_run.json =================================================================== --- system_test/testcase_to_run.json (revision 1397151) +++ system_test/testcase_to_run.json (working copy) @@ -1,6 +1,5 @@ { - "ReplicaBasicTest" : [ - "testcase_0001", - "testcase_1" + "MirrorMakerTest" : [ + "testcase_5002" ] } Index: system_test/metrics.json =================================================================== --- system_test/metrics.json (revision 1397151) +++ system_test/metrics.json (working copy) @@ -8,114 +8,6 @@ "y_label": "requests-per-sec", "bean_name": "kafka.network:type=RequestMetrics,name=Produce-RequestsPerSec", "attributes": "OneMinuteRate" - }, - { - "graph_name": "Produce-Request-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Produce-TotalTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "Produce-Request-Remote-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Produce-RemoteTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "Fetch-Consumer-Request-Rate", - "y_label": "requests-per-sec", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Consumer-RequestsPerSec", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "Fetch-Consumer-Request-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Consumer-TotalTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "Fetch-Consumer-Request-Remote-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Consumer-RemoteTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "Fetch-Follower-Request-Rate", - "y_label": "requests-per-sec", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Follower-RequestsPerSec", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "Fetch-Follower-Request-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Follower-TotalTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "Fetch-Follower-Request-Remote-Time", - "y_label": "ns,ns", - "bean_name": "kafka.network:type=RequestMetrics,name=Fetch-Follower-RemoteTimeNs", - "attributes": "Mean,99thPercentile" - }, - { - "graph_name": "ProducePurgatoryExpirationRate", - "y_label": "expirations-per-sec", - "bean_name": "kafka.server:type=DelayedProducerRequestMetrics,name=AllExpiresPerSecond", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "FetchConsumerPurgatoryExpirationRate", - "y_label": "expirations-per-sec", - "bean_name": "kafka.server:type=DelayedFetchRequestMetrics,name=ConsumerExpiresPerSecond", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "FetchFollowerPurgatoryExpirationRate", - "y_label": "expirations-per-sec", - "bean_name": "kafka.server:type=DelayedFetchRequestMetrics,name=FollowerExpiresPerSecond", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "ProducePurgatoryQueueSize", - "y_label": "size", - "bean_name": "kafka.server:type=ProducerRequestPurgatory,name=NumDelayedRequests", - "attributes": "Value" - }, - { - "graph_name": "FetchPurgatoryQueueSize", - "y_label": "size", - "bean_name": "kafka.server:type=FetchRequestPurgatory,name=NumDelayedRequests", - "attributes": "Value" - }, - { - "graph_name": "ControllerLeaderElectionRateAndTime", - "y_label": "elections-per-sec,ms,ms", - "bean_name": "kafka.server:type=ControllerStat,name=LeaderElectionRateAndTimeMs", - "attributes": "OneMinuteRate,Mean,99thPercentile" - }, - { - "graph_name": "LogFlushRateAndTime", - "y_label": "flushes-per-sec,ms,ms", - "bean_name": "kafka.message:type=LogFlushStats,name=LogFlushRateAndTimeMs", - "attributes": "OneMinuteRate,Mean,99thPercentile" - }, - { - "graph_name": "AllBytesOutRate", - "y_label": "bytes-per-sec", - "bean_name": "kafka.server:type=BrokerTopicMetrics,name=AllTopicsBytesOutPerSec", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "AllBytesInRate", - "y_label": "bytes-per-sec", - "bean_name": "kafka.server:type=BrokerTopicMetrics,name=AllTopicsBytesInPerSec", - "attributes": "OneMinuteRate" - }, - { - "graph_name": "AllMessagesInRate", - "y_label": "messages-per-sec", - "bean_name": "kafka.server:type=BrokerTopicMetrics,name=AllTopicsMessagesInPerSec", - "attributes": "OneMinuteRate" } ] }, @@ -123,12 +15,6 @@ "role": "producer_performance", "graphs": [ { - "graph_name": "ProduceRequestRateAndTime", - "y_label": "requests-per-sec,ms,ms", - "bean_name": "kafka.producer:type=ProducerRequestStat,name=ProduceRequestRateAndTimeMs", - "attributes": "OneMinuteRate,Mean,99thPercentile" - }, - { "graph_name": "ProduceRequestSize", "y_label": "bytes,bytes", "bean_name": "kafka.producer:type=ProducerRequestStat,name=ProducerRequestSize", @@ -140,18 +26,6 @@ "role": "console_consumer", "graphs": [ { - "graph_name": "FetchRequestRateAndTime", - "y_label": "requests-per-sec,ms,ms", - "bean_name": "kafka.consumer:type=FetchRequestAndResponseStat,name=FetchRequestRateAndTimeMs", - "attributes": "OneMinuteRate,Mean,99thPercentile" - }, - { - "graph_name": "FetchResponseSize", - "y_label": "bytes,bytes", - "bean_name": "kafka.consumer:type=FetchRequestAndResponseStat,name=FetchResponseSize", - "attributes": "Mean,99thPercentile" - }, - { "graph_name": "ConsumedMessageRate", "y_label": "messages-per-sec", "bean_name": "kafka.consumer:type=ConsumerTopicStat,name=AllTopicsMessagesPerSec",