diff --git a/system_test/mirror_maker_testsuite/cluster_config.json b/system_test/mirror_maker_testsuite/cluster_config.json index b679eea..5b908ff 100644 --- a/system_test/mirror_maker_testsuite/cluster_config.json +++ b/system_test/mirror_maker_testsuite/cluster_config.json @@ -121,6 +121,16 @@ "kafka_home": "default", "java_home": "default", "jmx_port": "9112" + }, + + { + "entity_id": "13", + "hostname": "localhost", + "role": "console_consumer", + "cluster_name":"source", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9113" } ] } diff --git a/system_test/mirror_maker_testsuite/config/mirror_producer.properties b/system_test/mirror_maker_testsuite/config/mirror_producer.properties index 7db5bfc..ffa5e1c 100644 --- a/system_test/mirror_maker_testsuite/config/mirror_producer.properties +++ b/system_test/mirror_maker_testsuite/config/mirror_producer.properties @@ -2,4 +2,5 @@ producer.type=async queue.enqueue.timeout.ms=-1 broker.list=localhost:9094 compression.codec=0 - +message.send.max.retries=3 +request.required.acks=1 diff --git a/system_test/mirror_maker_testsuite/mirror_maker_test.py b/system_test/mirror_maker_testsuite/mirror_maker_test.py index 098f531..4dc3cdf 100644 --- a/system_test/mirror_maker_testsuite/mirror_maker_test.py +++ b/system_test/mirror_maker_testsuite/mirror_maker_test.py @@ -168,13 +168,13 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils): self.log_message("starting mirror makers") kafka_system_test_utils.start_mirror_makers(self.systemTestEnv, self.testcaseEnv) - self.anonLogger.info("sleeping for 5s") - time.sleep(5) + self.anonLogger.info("sleeping for 10s") + time.sleep(10) - self.log_message("creating topics") - kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) - self.anonLogger.info("sleeping for 5s") - time.sleep(5) + #self.log_message("creating topics") + #kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) + #self.anonLogger.info("sleeping for 5s") + #time.sleep(5) # ============================================= @@ -192,7 +192,7 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils): # ============================================= i = 1 numIterations = int(self.testcaseEnv.testcaseArgumentsDict["num_iteration"]) - bouncedEntityDownTimeSec = 1 + bouncedEntityDownTimeSec = 15 try: bouncedEntityDownTimeSec = int(self.testcaseEnv.testcaseArgumentsDict["bounced_entity_downtime_sec"]) except: @@ -254,6 +254,17 @@ class MirrorMakerTest(ReplicationUtils, SetupUtils): self.testcaseEnv.lock.release() time.sleep(2) + self.anonLogger.info("sleeping for 15s") + time.sleep(15) + self.anonLogger.info("terminate Mirror Maker") + cmdStr = "ps auxw | grep Mirror | grep -v grep | tr -s ' ' | cut -f2 -d ' ' | xargs kill -15" + subproc = system_test_utils.sys_call_return_subproc(cmdStr) + for line in subproc.stdout.readlines(): + line = line.rstrip('\n') + self.anonLogger.info("#### ["+line+"]") + self.anonLogger.info("sleeping for 15s") + time.sleep(15) + # ============================================= # starting consumer # ============================================= diff --git a/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json b/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json index abb500b..287cab9 100644 --- a/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json @@ -56,6 +56,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -65,6 +67,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -74,6 +78,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -83,6 +89,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -92,6 +100,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -101,6 +111,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -114,6 +126,7 @@ "message": "500", "request-num-acks": "-1", "sync":"true", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, @@ -131,6 +144,15 @@ "log_filename": "mirror_maker_12.log", "mirror_consumer_config_filename": "mirror_consumer_12.properties", "mirror_producer_config_filename": "mirror_producer_12.properties" + }, + + { + "entity_id": "13", + "topic": "test_1", + "group.id": "mytestgroup", + "consumer-timeout-ms": "10000", + "log_filename": "console_consumer_13.log", + "config_filename": "console_consumer_13.properties" } ] } diff --git a/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json b/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json index 8d25bad..5457eb1 100644 --- a/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json @@ -56,6 +56,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -65,6 +67,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -74,6 +78,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -83,6 +89,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -92,6 +100,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -101,6 +111,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -114,6 +126,7 @@ "message": "100", "request-num-acks": "-1", "sync":"true", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, @@ -131,6 +144,15 @@ "log_filename": "mirror_maker_12.log", "mirror_consumer_config_filename": "mirror_consumer_12.properties", "mirror_producer_config_filename": "mirror_producer_12.properties" + }, + + { + "entity_id": "13", + "topic": "test_1", + "group.id": "mytestgroup", + "consumer-timeout-ms": "10000", + "log_filename": "console_consumer_13.log", + "config_filename": "console_consumer_13.properties" } ] } diff --git a/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json b/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json index b9d72a1..98fefee 100644 --- a/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5003/testcase_5003_properties.json @@ -57,6 +57,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -66,6 +68,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -75,6 +79,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -84,6 +90,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -93,6 +101,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -102,6 +112,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -115,6 +127,7 @@ "message": "100", "request-num-acks": "-1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, diff --git a/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json b/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json index b72f265..6067b12 100644 --- a/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5004/testcase_5004_properties.json @@ -57,6 +57,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -66,6 +68,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -75,6 +79,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -84,6 +90,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -93,6 +101,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -102,6 +112,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -115,6 +127,7 @@ "message": "100", "request-num-acks": "1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, diff --git a/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json b/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json index 83ca969..58483ad 100644 --- a/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5005/testcase_5005_properties.json @@ -57,6 +57,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -66,6 +68,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -75,6 +79,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -84,6 +90,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -93,6 +101,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -102,6 +112,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -115,6 +127,7 @@ "message": "100", "request-num-acks": "-1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, @@ -127,6 +140,7 @@ "message": "100", "request-num-acks": "-1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_11.log", "config_filename": "producer_performance_11.properties" }, diff --git a/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json b/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json index fe32712..1d9190c 100644 --- a/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json +++ b/system_test/mirror_maker_testsuite/testcase_5006/testcase_5006_properties.json @@ -57,6 +57,8 @@ "broker.id": "1", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_4_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_4.log", "config_filename": "kafka_server_4.properties" }, @@ -66,6 +68,8 @@ "broker.id": "2", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_5_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_5.log", "config_filename": "kafka_server_5.properties" }, @@ -75,6 +79,8 @@ "broker.id": "3", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_6_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_6.log", "config_filename": "kafka_server_6.properties" }, @@ -84,6 +90,8 @@ "broker.id": "4", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_7_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_7.log", "config_filename": "kafka_server_7.properties" }, @@ -93,6 +101,8 @@ "broker.id": "5", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_8_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_8.log", "config_filename": "kafka_server_8.properties" }, @@ -102,6 +112,8 @@ "broker.id": "6", "log.segment.bytes": "10240", "log.dir": "/tmp/kafka_server_9_logs", + "default.replication.factor": "3", + "num.partitions": "5", "log_filename": "kafka_server_9.log", "config_filename": "kafka_server_9.properties" }, @@ -115,6 +127,7 @@ "message": "100", "request-num-acks": "1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_10.log", "config_filename": "producer_performance_10.properties" }, @@ -127,6 +140,7 @@ "message": "100", "request-num-acks": "1", "sync":"false", + "producer-num-retries":"5", "log_filename": "producer_performance_11.log", "config_filename": "producer_performance_11.properties" },