Index: system_test/migration_tool_testsuite/0.7/bin/kafka-run-class.sh =================================================================== --- system_test/migration_tool_testsuite/0.7/bin/kafka-run-class.sh (revision 0) +++ system_test/migration_tool_testsuite/0.7/bin/kafka-run-class.sh (revision 0) @@ -0,0 +1,66 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname [opts]" + exit 1 +fi + +base_dir=$(dirname $0)/../../../.. +kafka_07_lib_dir=$(dirname $0)/../lib + +# 0.8 - scala jars +for file in $base_dir/project/boot/scala-2.8.0/lib/*.jar; +do + CLASSPATH=$CLASSPATH:$file +done + +# 0.7 - kafka-0.7.jar, zkclient-0.1.jar, kafka-perf-0.7.0.jar +for file in ${kafka_07_lib_dir}/*.jar; +do + CLASSPATH=$CLASSPATH:$file +done + +# 0.8 - metrics jar +for file in $base_dir/core/lib/metrics*.jar; +do + CLASSPATH=$CLASSPATH:$file +done + +# 0.8 - misc jars +for file in $base_dir/core/lib_managed/scala_2.8.0/compile/*.jar; +do + if [ ${file##*/} != "sbt-launch.jar" ]; then + CLASSPATH=$CLASSPATH:$file + fi +done +if [ -z "$KAFKA_JMX_OPTS" ]; then + KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi +if [ -z "$KAFKA_OPTS" ]; then + KAFKA_OPTS="-Xmx512M -server -Dlog4j.configuration=file:$base_dir/config/log4j.properties" +fi +if [ $JMX_PORT ]; then + KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT " +fi +if [ -z "$JAVA_HOME" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +$JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH $@ Index: system_test/migration_tool_testsuite/0.7/bin/zookeeper-server-start.sh =================================================================== --- system_test/migration_tool_testsuite/0.7/bin/zookeeper-server-start.sh (revision 0) +++ system_test/migration_tool_testsuite/0.7/bin/zookeeper-server-start.sh (revision 0) @@ -0,0 +1,23 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -ne 1 ]; +then + echo "USAGE: $0 zookeeper.properties" + exit 1 +fi + +$(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ Index: system_test/migration_tool_testsuite/cluster_config.json =================================================================== --- system_test/migration_tool_testsuite/cluster_config.json (revision 0) +++ system_test/migration_tool_testsuite/cluster_config.json (revision 0) @@ -0,0 +1,103 @@ +{ + "cluster_config": [ + { + "entity_id": "0", + "hostname": "localhost", + "role": "zookeeper", + "cluster_name":"source", + "kafka_home": "system_test/migration_tool_testsuite/0.7", + "java_home": "default", + "jmx_port": "9990" + }, + { + "entity_id": "1", + "hostname": "localhost", + "role": "broker", + "cluster_name":"source", + "kafka_home": "system_test/migration_tool_testsuite/0.7", + "java_home": "default", + "jmx_port": "9991" + }, + { + "entity_id": "2", + "hostname": "localhost", + "role": "broker", + "cluster_name":"source", + "kafka_home": "system_test/migration_tool_testsuite/0.7", + "java_home": "default", + "jmx_port": "9992" + }, + { + "entity_id": "3", + "hostname": "localhost", + "role": "broker", + "cluster_name":"source", + "kafka_home": "system_test/migration_tool_testsuite/0.7", + "java_home": "default", + "jmx_port": "9993" + }, + { + "entity_id": "4", + "hostname": "localhost", + "role": "broker", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9994" + }, + { + "entity_id": "5", + "hostname": "localhost", + "role": "broker", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9995" + }, + { + "entity_id": "6", + "hostname": "localhost", + "role": "broker", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9996" + }, + { + "entity_id": "7", + "hostname": "localhost", + "role": "producer_performance", + "cluster_name":"source", + "kafka_home": "system_test/migration_tool_testsuite/0.7", + "java_home": "/export/apps/jdk/JDK-1_6_0_27", + "jmx_port": "9997" + }, + { + "entity_id": "8", + "hostname": "localhost", + "role": "console_consumer", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9998" + }, + { + "entity_id": "9", + "hostname": "localhost", + "role": "zookeeper", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9999" + }, + { + "entity_id": "10", + "hostname": "localhost", + "role": "migration_tool", + "cluster_name":"target", + "kafka_home": "default", + "java_home": "default", + "jmx_port": "9890" + } + ] +} Index: system_test/migration_tool_testsuite/config/migration_consumer.properties =================================================================== --- system_test/migration_tool_testsuite/config/migration_consumer.properties (revision 0) +++ system_test/migration_tool_testsuite/config/migration_consumer.properties (revision 0) @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# zk connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zk.connect=127.0.0.1:2188 + +# timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +#consumer group id +groupid=test-consumer-group + +#consumer timeout +#consumer.timeout.ms=5000 Index: system_test/migration_tool_testsuite/config/migration_producer.properties =================================================================== --- system_test/migration_tool_testsuite/config/migration_producer.properties (revision 0) +++ system_test/migration_tool_testsuite/config/migration_producer.properties (revision 0) @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.producer.ProducerConfig for more details + +############################# Producer Basics ############################# + +# need to set either broker.list or zk.connect + +# configure brokers statically +# format: host1:port1,host2:port2 ... +broker.list=localhost:9094,localhost:9095,localhost:9096 + +# discover brokers from ZK +#zk.connect= + +# zookeeper session timeout; default is 6000 +#zk.sessiontimeout.ms= + +# the max time that the client waits to establish a connection to zookeeper; default is 6000 +#zk.connectiontimeout.ms + +# name of the partitioner class for partitioning events; default partition spreads data randomly +#partitioner.class= + +# specifies whether the messages are sent asynchronously (async) or synchronously (sync) +producer.type=sync + +# specify the compression codec for all data generated: 0: no compression, 1: gzip +compression.codec=0 + +# message encoder +serializer.class=kafka.serializer.DefaultEncoder + +# allow topic level compression +#compressed.topics= + +# max message size; messages larger than that size are discarded; default is 1000000 +#max.message.size= + + +############################# Async Producer ############################# +# maximum time, in milliseconds, for buffering data on the producer queue +#queue.time= + +# the maximum size of the blocking queue for buffering on the producer +#queue.size= + +# Timeout for event enqueue: +# 0: events will be enqueued immediately or dropped if the queue is full +# -ve: enqueue will block indefinitely if the queue is full +# +ve: enqueue will block up to this many milliseconds if the queue is full +#queue.enqueueTimeout.ms= + +# the number of messages batched at the producer +#batch.size= + +# the callback handler for one or multiple events +#callback.handler= + +# properties required to initialize the callback handler +#callback.handler.props= + +# the handler for events +#event.handler= + +# properties required to initialize the event handler +#event.handler.props= + Index: system_test/migration_tool_testsuite/config/server.properties =================================================================== --- system_test/migration_tool_testsuite/config/server.properties (revision 0) +++ system_test/migration_tool_testsuite/config/server.properties (revision 0) @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +brokerid=0 + +# Hostname the broker will advertise to consumers. If not set, kafka will use the value returned +# from InetAddress.getLocalHost(). If there are multiple interfaces getLocalHost +# may not be what you want. +#hostname= + + +############################# Socket Server Settings ############################# + +# The port the socket server listens on +port=9091 + +# The number of threads handling network requests +network.threads=2 + +# The number of threads doing disk I/O +io.threads=2 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer=1048576 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer=1048576 + +# The maximum size of a request that the socket server will accept (protection against OOM) +max.socket.request.bytes=104857600 + + +############################# Log Basics ############################# + +# The directory under which to store log files +log.dir=/tmp/kafka_server_logs + +# The number of logical partitions per topic per server. More partitions allow greater parallelism +# for consumption, but also mean more files. +num.partitions=5 + +# Overrides for for the default given by num.partitions on a per-topic basis +#topic.partition.count.map=topic1:3, topic2:4 + +############################# Log Flush Policy ############################# + +# The following configurations control the flush of data to disk. This is the most +# important performance knob in kafka. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data is at greater risk of loss in the event of a crash. +# 2. Latency: Data is not made available to consumers until it is flushed (which adds latency). +# 3. Throughput: The flush is generally the most expensive operation. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +log.flush.interval=10000 + +# The maximum amount of time a message can sit in a log before we force a flush +log.default.flush.interval.ms=1000 + +# Per-topic overrides for log.default.flush.interval.ms +#topic.flush.intervals.ms=topic1:1000, topic2:3000 + +# The interval (in ms) at which logs are checked to see if they need to be flushed to disk. +log.default.flush.scheduler.interval.ms=1000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion +log.retention.hours=168 + +# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining +# segments don't drop below log.retention.size. +#log.retention.size=1073741824 + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +#log.file.size=536870912 +#log.file.size=102400 +log.file.size=128 + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.cleanup.interval.mins=1 + +############################# Zookeeper ############################# + +# Enable connecting to zookeeper +enable.zookeeper=true + +# Zk connection string (see zk docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zk.connect=localhost:2181 + +# Timeout in ms for connecting to zookeeper +zk.connectiontimeout.ms=1000000 + +monitoring.period.secs=1 Index: system_test/migration_tool_testsuite/config/zookeeper.properties =================================================================== --- system_test/migration_tool_testsuite/config/zookeeper.properties (revision 0) +++ system_test/migration_tool_testsuite/config/zookeeper.properties (revision 0) @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# disable the per-ip limit on the number of connections since this is a non-production config +maxClientCnxns=0 Index: system_test/migration_tool_testsuite/__init__.py =================================================================== --- system_test/migration_tool_testsuite/__init__.py (revision 0) +++ system_test/migration_tool_testsuite/__init__.py (revision 0) @@ -0,0 +1 @@ + Index: system_test/migration_tool_testsuite/migration_tool_test.py =================================================================== --- system_test/migration_tool_testsuite/migration_tool_test.py (revision 0) +++ system_test/migration_tool_testsuite/migration_tool_test.py (revision 0) @@ -0,0 +1,322 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#!/usr/bin/env python + +# =================================== +# migration_tool_test.py +# =================================== + +import inspect +import logging +import os +import signal +import subprocess +import sys +import time +import traceback + +from system_test_env import SystemTestEnv +sys.path.append(SystemTestEnv.SYSTEM_TEST_UTIL_DIR) + +from setup_utils import SetupUtils +from replication_utils import ReplicationUtils +import system_test_utils +from testcase_env import TestcaseEnv + +# product specific: Kafka +import kafka_system_test_utils +import metrics + +class MigrationToolTest(ReplicationUtils, SetupUtils): + + testModuleAbsPathName = os.path.realpath(__file__) + testSuiteAbsPathName = os.path.abspath(os.path.dirname(testModuleAbsPathName)) + + def __init__(self, systemTestEnv): + + # SystemTestEnv - provides cluster level environment settings + # such as entity_id, hostname, kafka_home, java_home which + # are available in a list of dictionary named + # "clusterEntityConfigDictList" + self.systemTestEnv = systemTestEnv + + super(MigrationToolTest, self).__init__(self) + + # dict to pass user-defined attributes to logger argument: "extra" + d = {'name_of_class': self.__class__.__name__} + + def signal_handler(self, signal, frame): + self.log_message("Interrupt detected - User pressed Ctrl+c") + + # perform the necessary cleanup here when user presses Ctrl+c and it may be product specific + self.log_message("stopping all entities - please wait ...") + kafka_system_test_utils.stop_all_remote_running_processes(self.systemTestEnv, self.testcaseEnv) + sys.exit(1) + + def runTest(self): + + # ====================================================================== + # get all testcase directories under this testsuite + # ====================================================================== + testCasePathNameList = system_test_utils.get_dir_paths_with_prefix( + self.testSuiteAbsPathName, SystemTestEnv.SYSTEM_TEST_CASE_PREFIX) + testCasePathNameList.sort() + + # ============================================================= + # launch each testcase one by one: testcase_1, testcase_2, ... + # ============================================================= + for testCasePathName in testCasePathNameList: + + skipThisTestCase = False + + try: + # ====================================================================== + # A new instance of TestcaseEnv to keep track of this testcase's env vars + # and initialize some env vars as testCasePathName is available now + # ====================================================================== + self.testcaseEnv = TestcaseEnv(self.systemTestEnv, self) + self.testcaseEnv.testSuiteBaseDir = self.testSuiteAbsPathName + self.testcaseEnv.initWithKnownTestCasePathName(testCasePathName) + self.testcaseEnv.testcaseArgumentsDict = self.testcaseEnv.testcaseNonEntityDataDict["testcase_args"] + + # ====================================================================== + # SKIP if this case is IN testcase_to_skip.json or NOT IN testcase_to_run.json + # ====================================================================== + testcaseDirName = self.testcaseEnv.testcaseResultsDict["_test_case_name"] + + if self.systemTestEnv.printTestDescriptionsOnly: + self.testcaseEnv.printTestCaseDescription(testcaseDirName) + continue + elif self.systemTestEnv.isTestCaseToSkip(self.__class__.__name__, testcaseDirName): + self.log_message("Skipping : " + testcaseDirName) + skipThisTestCase = True + continue + else: + self.testcaseEnv.printTestCaseDescription(testcaseDirName) + system_test_utils.setup_remote_hosts_with_testcase_level_cluster_config(self.systemTestEnv, testCasePathName) + + # ============================================================================== # + # ============================================================================== # + # Product Specific Testing Code Starts Here: # + # ============================================================================== # + # ============================================================================== # + + # initialize self.testcaseEnv with user-defined environment variables (product specific) + self.testcaseEnv.userDefinedEnvVarDict["zkConnectStr"] = "" + self.testcaseEnv.userDefinedEnvVarDict["stopBackgroundProducer"] = False + self.testcaseEnv.userDefinedEnvVarDict["backgroundProducerStopped"] = False + + # initialize signal handler + signal.signal(signal.SIGINT, self.signal_handler) + + # create "LOCAL" log directories for metrics, dashboards for each entity under this testcase + # for collecting logs from remote machines + kafka_system_test_utils.generate_testcase_log_dirs(self.systemTestEnv, self.testcaseEnv) + + # TestcaseEnv.testcaseConfigsList initialized by reading testcase properties file: + # system_test/_testsuite/testcase_/testcase__properties.json + self.testcaseEnv.testcaseConfigsList = system_test_utils.get_json_list_data( + self.testcaseEnv.testcasePropJsonPathName) + + # TestcaseEnv - initialize producer & consumer config / log file pathnames + kafka_system_test_utils.init_entity_props(self.systemTestEnv, self.testcaseEnv) + + # clean up data directories specified in zookeeper.properties and kafka_server_.properties + kafka_system_test_utils.cleanup_data_at_remote_hosts(self.systemTestEnv, self.testcaseEnv) + + # generate remote hosts log/config dirs if not exist + kafka_system_test_utils.generate_testcase_log_dirs_in_remote_hosts(self.systemTestEnv, self.testcaseEnv) + + # generate properties files for zookeeper, kafka, producer, consumer and mirror-maker: + # 1. copy system_test/_testsuite/config/*.properties to + # system_test/_testsuite/testcase_/config/ + # 2. update all properties files in system_test/_testsuite/testcase_/config + # by overriding the settings specified in: + # system_test/_testsuite/testcase_/testcase__properties.json + kafka_system_test_utils.generate_overriden_props_files(self.testSuiteAbsPathName, + self.testcaseEnv, self.systemTestEnv) + + # ============================================= + # preparing all entities to start the test + # ============================================= + self.log_message("starting zookeepers") + kafka_system_test_utils.start_zookeepers(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 2s") + time.sleep(2) + + self.log_message("starting brokers") + kafka_system_test_utils.start_brokers(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 5s") + time.sleep(5) + + self.log_message("starting migration tool") + kafka_system_test_utils.start_migration_tool(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 5s") + time.sleep(5) + + self.log_message("creating topics") + kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 5s") + time.sleep(5) + + # ============================================= + # starting producer + # ============================================= + self.log_message("starting producer in the background") + kafka_system_test_utils.start_producer_performance(self.systemTestEnv, self.testcaseEnv, True) + msgProducingFreeTimeSec = self.testcaseEnv.testcaseArgumentsDict["message_producing_free_time_sec"] + self.anonLogger.info("sleeping for " + msgProducingFreeTimeSec + " sec to produce some messages") + time.sleep(int(msgProducingFreeTimeSec)) + + # ============================================= + # A while-loop to bounce leader as specified + # by "num_iterations" in testcase_n_properties.json + # ============================================= + i = 1 + numIterations = int(self.testcaseEnv.testcaseArgumentsDict["num_iteration"]) + while i <= numIterations: + + self.log_message("Iteration " + str(i) + " of " + str(numIterations)) + + self.log_message("looking up leader") + leaderDict = kafka_system_test_utils.get_leader_elected_log_line( + self.systemTestEnv, self.testcaseEnv, self.leaderAttributesDict) + + # ========================== + # leaderDict looks like this: + # ========================== + #{'entity_id': u'3', + # 'partition': '0', + # 'timestamp': 1345050255.8280001, + # 'hostname': u'localhost', + # 'topic': 'test_1', + # 'brokerid': '3'} + + # ============================================= + # validate to see if leader election is successful + # ============================================= + self.log_message("validating leader election") + result = kafka_system_test_utils.validate_leader_election_successful( + self.testcaseEnv, leaderDict, self.testcaseEnv.validationStatusDict) + + # ============================================= + # trigger leader re-election by stopping leader + # to get re-election latency + # ============================================= + bounceLeaderFlag = self.testcaseEnv.testcaseArgumentsDict["bounce_leader"] + self.log_message("bounce_leader flag : " + bounceLeaderFlag) + if (bounceLeaderFlag.lower() == "true"): + reelectionLatency = kafka_system_test_utils.get_reelection_latency( + self.systemTestEnv, self.testcaseEnv, leaderDict, self.leaderAttributesDict) + latencyKeyName = "Leader Election Latency - iter " + str(i) + " brokerid " + leaderDict["brokerid"] + self.testcaseEnv.validationStatusDict[latencyKeyName] = str("{0:.2f}".format(reelectionLatency * 1000)) + " ms" + + # ============================================= + # starting previously terminated broker + # ============================================= + if bounceLeaderFlag.lower() == "true": + self.log_message("starting the previously terminated broker") + stoppedLeaderEntityId = leaderDict["entity_id"] + kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedLeaderEntityId) + + self.anonLogger.info("sleeping for 15s") + time.sleep(15) + i += 1 + # while loop + + # ============================================= + # tell producer to stop + # ============================================= + self.testcaseEnv.lock.acquire() + self.testcaseEnv.userDefinedEnvVarDict["stopBackgroundProducer"] = True + time.sleep(1) + self.testcaseEnv.lock.release() + time.sleep(1) + + # ============================================= + # wait for producer thread's update of + # "backgroundProducerStopped" to be "True" + # ============================================= + while 1: + self.testcaseEnv.lock.acquire() + self.logger.info("status of backgroundProducerStopped : [" + \ + str(self.testcaseEnv.userDefinedEnvVarDict["backgroundProducerStopped"]) + "]", extra=self.d) + if self.testcaseEnv.userDefinedEnvVarDict["backgroundProducerStopped"]: + time.sleep(1) + self.logger.info("all producer threads completed", extra=self.d) + break + time.sleep(1) + self.testcaseEnv.lock.release() + time.sleep(2) + + #print "\n\n#### sleeping for 30 min ...\n\n" + #time.sleep(1800) + + # ============================================= + # starting consumer + # ============================================= + self.log_message("starting consumer in the background") + kafka_system_test_utils.start_console_consumer(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 20s") + time.sleep(20) + + # ============================================= + # this testcase is completed - stop all entities + # ============================================= + self.log_message("stopping all entities") + for entityId, parentPid in self.testcaseEnv.entityBrokerParentPidDict.items(): + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, entityId, parentPid) + + for entityId, parentPid in self.testcaseEnv.entityZkParentPidDict.items(): + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, entityId, parentPid) + + # make sure all entities are stopped + kafka_system_test_utils.ps_grep_terminate_running_entity(self.systemTestEnv) + + # ============================================= + # collect logs from remote hosts + # ============================================= + kafka_system_test_utils.collect_logs_from_remote_hosts(self.systemTestEnv, self.testcaseEnv) + + # ============================================= + # validate the data matched and checksum + # ============================================= + self.log_message("validating data matched") + kafka_system_test_utils.validate_07_08_migrated_data_matched(self.systemTestEnv, self.testcaseEnv) + kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) + + # ============================================= + # draw graphs + # ============================================= + metrics.draw_all_graphs(self.systemTestEnv.METRICS_PATHNAME, + self.testcaseEnv, + self.systemTestEnv.clusterEntityConfigDictList) + + # build dashboard, one for each role + metrics.build_all_dashboards(self.systemTestEnv.METRICS_PATHNAME, + self.testcaseEnv.testCaseDashboardsDir, + self.systemTestEnv.clusterEntityConfigDictList) + + except Exception as e: + self.log_message("Exception while running test {0}".format(e)) + traceback.print_exc() + + finally: + if not skipThisTestCase and not self.systemTestEnv.printTestDescriptionsOnly: + self.log_message("stopping all entities - please wait ...") + kafka_system_test_utils.stop_all_remote_running_processes(self.systemTestEnv, self.testcaseEnv) + Index: system_test/migration_tool_testsuite/testcase_9001/testcase_9001_properties.json =================================================================== --- system_test/migration_tool_testsuite/testcase_9001/testcase_9001_properties.json (revision 0) +++ system_test/migration_tool_testsuite/testcase_9001/testcase_9001_properties.json (revision 0) @@ -0,0 +1,123 @@ +{ + "description": {"01":"To Test : 'Replication with Migration Tool'", + "02":"Set up 2 clusters such as : SOURCE => Migration Tool => TARGET", + "03":"Produce and consume messages to a single topic - single partition.", + "04":"This test sends messages to 3 replicas", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss in TARGET cluster.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 51200" + }, + "testcase_args": { + "bounce_leader": "false", + "replica_factor": "3", + "num_partition": "1", + "num_iteration": "1", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "30", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_0.log", + "config_filename": "zookeeper_0.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_1.log", + "config_filename": "kafka_server_1.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_2.log", + "config_filename": "kafka_server_2.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_3.log", + "config_filename": "kafka_server_3.properties" + }, + { + "entity_id": "4", + "port": "9094", + "brokerid": "4", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_4_logs", + "log_filename": "kafka_server_4.log", + "config_filename": "kafka_server_4.properties" + }, + { + "entity_id": "5", + "port": "9095", + "brokerid": "5", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_5_logs", + "log_filename": "kafka_server_5.log", + "config_filename": "kafka_server_5.properties" + }, + { + "entity_id": "6", + "port": "9096", + "brokerid": "6", + "log.file.size": "51200", + "log.dir": "/tmp/kafka_server_6_logs", + "log_filename": "kafka_server_6.log", + "config_filename": "kafka_server_6.properties" + }, + { + "entity_id": "7", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "500", + "request-num-acks": "-1", + "async": "false", + "log_filename": "producer_performance_7.log", + "config_filename": "producer_performance_7.properties" + }, + { + "entity_id": "8", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "log_filename": "console_consumer_8.log", + "formatter": "kafka.consumer.ChecksumMessageFormatter", + "config_filename": "console_consumer_8.properties" + }, + { + "entity_id": "9", + "clientPort": "2191", + "dataDir": "/tmp/zookeeper_9", + "log_filename": "zookeeper_9.log", + "config_filename": "zookeeper_9.properties" + }, + { + "entity_id": "10", + "whitelist": ".*", + "num.producers": "2", + "num.streams": "2", + "producer.config": "migration_tool_testsuite/config/migration_producer.properties", + "consumer.config": "migration_tool_testsuite/config/migration_consumer.properties", + "zkclient.01.jar": "migration_tool_testsuite/0.7/lib/zkclient-0.1.jar", + "kafka.07.jar" : "migration_tool_testsuite/0.7/lib/kafka-0.7.0.jar", + "log_filename": "migration_tool.log", + "config_filename": "migration_tool.properties" + } + ] +} Index: system_test/mirror_maker_testsuite/config/console_consumer.properties =================================================================== --- system_test/mirror_maker_testsuite/config/console_consumer.properties (revision 1401726) +++ system_test/mirror_maker_testsuite/config/console_consumer.properties (working copy) @@ -1,4 +0,0 @@ -zookeeper=local:2181 -topic=test_1 -from-beginning -consumer-timeout-ms=10000 Index: system_test/mirror_maker_testsuite/config/consumer.properties =================================================================== --- system_test/mirror_maker_testsuite/config/consumer.properties (revision 1401726) +++ system_test/mirror_maker_testsuite/config/consumer.properties (working copy) @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.consumer.ConsumerConfig for more details - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=127.0.0.1:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -#consumer group id -groupid=test-consumer-group - -#consumer timeout -#consumer.timeout.ms=5000 Index: system_test/mirror_maker_testsuite/config/log4j.properties =================================================================== --- system_test/mirror_maker_testsuite/config/log4j.properties (revision 1401726) +++ system_test/mirror_maker_testsuite/config/log4j.properties (working copy) @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -#log4j.appender.fileAppender=org.apache.log4j.FileAppender -#log4j.appender.fileAppender.File=kafka-request.log -#log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout -#log4j.appender.fileAppender.layout.ConversionPattern= %-4r [%t] %-5p %c %x - %m%n - - -# Turn on all our debugging info -#log4j.logger.kafka=INFO -#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG - -# to print message checksum from ProducerPerformance -log4j.logger.kafka.perf=DEBUG -log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG - -# to print message checksum from ProducerPerformance -log4j.logger.kafka.perf=DEBUG -log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG - Index: system_test/mirror_maker_testsuite/config/producer_performance.properties =================================================================== --- system_test/mirror_maker_testsuite/config/producer_performance.properties (revision 1401726) +++ system_test/mirror_maker_testsuite/config/producer_performance.properties (working copy) @@ -1,5 +0,0 @@ -topic=mytest -message-size=100 -thread=5 -compression-codec=0 -request-num-acks=-1 Index: system_test/mirror_maker_testsuite/config/producer.properties =================================================================== --- system_test/mirror_maker_testsuite/config/producer.properties (revision 1401726) +++ system_test/mirror_maker_testsuite/config/producer.properties (working copy) @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.producer.ProducerConfig for more details - -############################# Producer Basics ############################# - -# need to set either broker.list or zk.connect - -# configure brokers statically -# format: brokerid1:host1:port1,brokerid2:host2:port2 ... -#broker.list=0:localhost:9092 - -# discover brokers from ZK -zk.connect=localhost:2181 - -# zookeeper session timeout; default is 6000 -#zk.sessiontimeout.ms= - -# the max time that the client waits to establish a connection to zookeeper; default is 6000 -#zk.connectiontimeout.ms - -# name of the partitioner class for partitioning events; default partition spreads data randomly -#partitioner.class= - -# specifies whether the messages are sent asynchronously (async) or synchronously (sync) -producer.type=sync - -# specify the compression codec for all data generated: 0: no compression, 1: gzip -compression.codec=0 - -# message encoder -serializer.class=kafka.serializer.StringEncoder - -# allow topic level compression -#compressed.topics= - -# max message size; messages larger than that size are discarded; default is 1000000 -#max.message.size= - - -############################# Async Producer ############################# -# maximum time, in milliseconds, for buffering data on the producer queue -#queue.time= - -# the maximum size of the blocking queue for buffering on the producer -#queue.size= - -# Timeout for event enqueue: -# 0: events will be enqueued immediately or dropped if the queue is full -# -ve: enqueue will block indefinitely if the queue is full -# +ve: enqueue will block up to this many milliseconds if the queue is full -#queue.enqueueTimeout.ms= - -# the number of messages batched at the producer -#batch.size= - -# the callback handler for one or multiple events -#callback.handler= - -# properties required to initialize the callback handler -#callback.handler.props= - -# the handler for events -#event.handler= - -# properties required to initialize the event handler -#event.handler.props= - Index: system_test/mirror_maker_testsuite/mirror_maker_test.py =================================================================== --- system_test/mirror_maker_testsuite/mirror_maker_test.py (revision 1401726) +++ system_test/mirror_maker_testsuite/mirror_maker_test.py (working copy) @@ -170,9 +170,6 @@ self.anonLogger.info("sleeping for 5s") time.sleep(5) - #print "#### sleeping for 30 min" - #time.sleep(1800) - self.log_message("creating topics") kafka_system_test_utils.create_topic(self.systemTestEnv, self.testcaseEnv) self.anonLogger.info("sleeping for 5s") @@ -189,7 +186,7 @@ time.sleep(int(msgProducingFreeTimeSec)) # ============================================= - # A while-loop to bounce leader as specified + # A while-loop to bounce mirror maker as specified # by "num_iterations" in testcase_n_properties.json # ============================================= i = 1 @@ -198,47 +195,26 @@ self.log_message("Iteration " + str(i) + " of " + str(numIterations)) - self.log_message("looking up leader") - leaderDict = kafka_system_test_utils.get_leader_elected_log_line( - self.systemTestEnv, self.testcaseEnv, self.leaderAttributesDict) - - # ========================== - # leaderDict looks like this: - # ========================== - #{'entity_id': u'3', - # 'partition': '0', - # 'timestamp': 1345050255.8280001, - # 'hostname': u'localhost', - # 'topic': 'test_1', - # 'brokerid': '3'} - # ============================================= - # validate to see if leader election is successful + # Bounce Mirror Maker if specified in testcase config # ============================================= - self.log_message("validating leader election") - result = kafka_system_test_utils.validate_leader_election_successful( - self.testcaseEnv, leaderDict, self.testcaseEnv.validationStatusDict) - - # ============================================= - # trigger leader re-election by stopping leader - # to get re-election latency - # ============================================= - bounceLeaderFlag = self.testcaseEnv.testcaseArgumentsDict["bounce_leader"] - self.log_message("bounce_leader flag : " + bounceLeaderFlag) - if (bounceLeaderFlag.lower() == "true"): - reelectionLatency = kafka_system_test_utils.get_reelection_latency( - self.systemTestEnv, self.testcaseEnv, leaderDict, self.leaderAttributesDict) - latencyKeyName = "Leader Election Latency - iter " + str(i) + " brokerid " + leaderDict["brokerid"] - self.testcaseEnv.validationStatusDict[latencyKeyName] = str("{0:.2f}".format(reelectionLatency * 1000)) + " ms" - - # ============================================= - # starting previously terminated broker - # ============================================= - if bounceLeaderFlag.lower() == "true": - self.log_message("starting the previously terminated broker") - stoppedLeaderEntityId = leaderDict["entity_id"] - kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedLeaderEntityId) + bounceMirrorMaker = self.testcaseEnv.testcaseArgumentsDict["bounce_mirror_maker"] + self.log_message("bounce_mirror_maker flag : " + bounceMirrorMaker) + if (bounceMirrorMaker.lower() == "true"): + clusterConfigList = self.systemTestEnv.clusterEntityConfigDictList + mirrorMakerEntityIdList = system_test_utils.get_data_from_list_of_dicts( + clusterConfigList, "role", "mirror_maker", "entity_id") + + mirrorMakerPPid = self.testcaseEnv.entityMirrorMakerParentPidDict[mirrorMakerEntityIdList[0]] + self.log_message("stopping mirror maker : " + mirrorMakerPPid) + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, mirrorMakerEntityIdList[0], mirrorMakerPPid) + time.sleep(1) + + # starting previously terminated broker + self.log_message("starting the previously terminated mirror maker") + kafka_system_test_utils.start_mirror_makers(self.systemTestEnv, self.testcaseEnv) + self.anonLogger.info("sleeping for 15s") time.sleep(15) i += 1 @@ -294,13 +270,15 @@ # collect logs from remote hosts # ============================================= kafka_system_test_utils.collect_logs_from_remote_hosts(self.systemTestEnv, self.testcaseEnv) - + # ============================================= # validate the data matched and checksum # ============================================= self.log_message("validating data matched") - kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) + #kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) + kafka_system_test_utils.validate_simple_consumer_data_matched(self.systemTestEnv, self.testcaseEnv) kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) + kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv, "target") # ============================================= # draw graphs @@ -313,7 +291,7 @@ metrics.build_all_dashboards(self.systemTestEnv.METRICS_PATHNAME, self.testcaseEnv.testCaseDashboardsDir, self.systemTestEnv.clusterEntityConfigDictList) - + except Exception as e: self.log_message("Exception while running test {0}".format(e)) traceback.print_exc() Index: system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json =================================================================== --- system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json (revision 1401726) +++ system_test/mirror_maker_testsuite/testcase_5001/testcase_5001_properties.json (working copy) @@ -11,6 +11,7 @@ }, "testcase_args": { "bounce_leader": "false", + "bounce_mirror_maker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json =================================================================== --- system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json (revision 0) +++ system_test/mirror_maker_testsuite/testcase_5002/testcase_5002_properties.json (revision 0) @@ -0,0 +1,136 @@ +{ + "description": {"01":"Replication with Mirror Maker => Bounce Mirror Maker", + "02":"Set up 2 clusters such as : SOURCE => MirrorMaker => TARGET", + "03":"Set up 2-node Zk cluster for both SOURCE & TARGET", + "04":"Produce and consume messages to a single topic - single partition.", + "05":"This test sends messages to 3 replicas", + "06":"At the end it verifies the log size and contents", + "07":"Use a consumer to verify no message loss in TARGET cluster.", + "08":"Producer dimensions : mode:sync, acks:-1, comp:0", + "09":"Log segment size : 10240" + }, + "testcase_args": { + "bounce_leader": "false", + "bounce_mirror_maker": "true", + "replica_factor": "3", + "num_partition": "1", + "num_iteration": "1", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2108", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_0.log", + "config_filename": "zookeeper_0.properties" + }, + { + "entity_id": "1", + "clientPort": "2118", + "dataDir": "/tmp/zookeeper_1", + "log_filename": "zookeeper_1.log", + "config_filename": "zookeeper_1.properties" + }, + + { + "entity_id": "2", + "clientPort": "2128", + "dataDir": "/tmp/zookeeper_2", + "log_filename": "zookeeper_2.log", + "config_filename": "zookeeper_2.properties" + }, + { + "entity_id": "3", + "clientPort": "2138", + "dataDir": "/tmp/zookeeper_3", + "log_filename": "zookeeper_3.log", + "config_filename": "zookeeper_3.properties" + }, + + { + "entity_id": "4", + "port": "9091", + "brokerid": "1", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_4_logs", + "log_filename": "kafka_server_4.log", + "config_filename": "kafka_server_4.properties" + }, + { + "entity_id": "5", + "port": "9092", + "brokerid": "2", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_5_logs", + "log_filename": "kafka_server_5.log", + "config_filename": "kafka_server_5.properties" + }, + { + "entity_id": "6", + "port": "9093", + "brokerid": "3", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_6_logs", + "log_filename": "kafka_server_6.log", + "config_filename": "kafka_server_6.properties" + }, + { + "entity_id": "7", + "port": "9094", + "brokerid": "4", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_7_logs", + "log_filename": "kafka_server_7.log", + "config_filename": "kafka_server_7.properties" + }, + { + "entity_id": "8", + "port": "9095", + "brokerid": "5", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_8_logs", + "log_filename": "kafka_server_8.log", + "config_filename": "kafka_server_8.properties" + }, + { + "entity_id": "9", + "port": "9096", + "brokerid": "6", + "log.file.size": "10240", + "log.dir": "/tmp/kafka_server_9_logs", + "log_filename": "kafka_server_9.log", + "config_filename": "kafka_server_9.properties" + }, + + { + "entity_id": "10", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance_10.log", + "config_filename": "producer_performance_10.properties" + }, + { + "entity_id": "11", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "log_filename": "console_consumer_11.log", + "config_filename": "console_consumer_11.properties" + }, + + { + "entity_id": "12", + "log_filename": "mirror_maker_12.log", + "mirror_consumer_config_filename": "mirror_consumer_12.properties", + "mirror_producer_config_filename": "mirror_producer_12.properties" + } + ] +} Index: system_test/replication_testsuite/config/console_consumer.properties =================================================================== --- system_test/replication_testsuite/config/console_consumer.properties (revision 1401726) +++ system_test/replication_testsuite/config/console_consumer.properties (working copy) @@ -1,4 +0,0 @@ -zookeeper=local:2181 -topic=test_1 -from-beginning -consumer-timeout-ms=10000 Index: system_test/replication_testsuite/config/consumer.properties =================================================================== --- system_test/replication_testsuite/config/consumer.properties (revision 1401726) +++ system_test/replication_testsuite/config/consumer.properties (working copy) @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.consumer.ConsumerConfig for more details - -# zk connection string -# comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" -zk.connect=127.0.0.1:2181 - -# timeout in ms for connecting to zookeeper -zk.connectiontimeout.ms=1000000 - -#consumer group id -groupid=test-consumer-group - -#consumer timeout -#consumer.timeout.ms=5000 Index: system_test/replication_testsuite/config/log4j.properties =================================================================== --- system_test/replication_testsuite/config/log4j.properties (revision 1401726) +++ system_test/replication_testsuite/config/log4j.properties (working copy) @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -log4j.rootLogger=INFO, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - -#log4j.appender.fileAppender=org.apache.log4j.FileAppender -#log4j.appender.fileAppender.File=kafka-request.log -#log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout -#log4j.appender.fileAppender.layout.ConversionPattern= %-4r [%t] %-5p %c %x - %m%n - - -# Turn on all our debugging info -#log4j.logger.kafka=INFO -#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG - -# to print message checksum from ProducerPerformance -log4j.logger.kafka.perf=DEBUG -log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG - -# to print message checksum from ProducerPerformance -log4j.logger.kafka.perf=DEBUG -log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG - Index: system_test/replication_testsuite/config/producer_performance.properties =================================================================== --- system_test/replication_testsuite/config/producer_performance.properties (revision 1401726) +++ system_test/replication_testsuite/config/producer_performance.properties (working copy) @@ -1,5 +0,0 @@ -topic=mytest -message-size=100 -thread=5 -compression-codec=0 -request-num-acks=-1 Index: system_test/replication_testsuite/config/producer.properties =================================================================== --- system_test/replication_testsuite/config/producer.properties (revision 1401726) +++ system_test/replication_testsuite/config/producer.properties (working copy) @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# see kafka.producer.ProducerConfig for more details - -############################# Producer Basics ############################# - -# need to set either broker.list or zk.connect - -# configure brokers statically -# format: brokerid1:host1:port1,brokerid2:host2:port2 ... -#broker.list=0:localhost:9092 - -# discover brokers from ZK -zk.connect=localhost:2181 - -# zookeeper session timeout; default is 6000 -#zk.sessiontimeout.ms= - -# the max time that the client waits to establish a connection to zookeeper; default is 6000 -#zk.connectiontimeout.ms - -# name of the partitioner class for partitioning events; default partition spreads data randomly -#partitioner.class= - -# specifies whether the messages are sent asynchronously (async) or synchronously (sync) -producer.type=sync - -# specify the compression codec for all data generated: 0: no compression, 1: gzip -compression.codec=0 - -# message encoder -serializer.class=kafka.serializer.StringEncoder - -# allow topic level compression -#compressed.topics= - -# max message size; messages larger than that size are discarded; default is 1000000 -#max.message.size= - - -############################# Async Producer ############################# -# maximum time, in milliseconds, for buffering data on the producer queue -#queue.time= - -# the maximum size of the blocking queue for buffering on the producer -#queue.size= - -# Timeout for event enqueue: -# 0: events will be enqueued immediately or dropped if the queue is full -# -ve: enqueue will block indefinitely if the queue is full -# +ve: enqueue will block up to this many milliseconds if the queue is full -#queue.enqueueTimeout.ms= - -# the number of messages batched at the producer -#batch.size= - -# the callback handler for one or multiple events -#callback.handler= - -# properties required to initialize the callback handler -#callback.handler.props= - -# the handler for events -#event.handler= - -# properties required to initialize the event handler -#event.handler.props= - Index: system_test/replication_testsuite/replica_basic_test.py =================================================================== --- system_test/replication_testsuite/replica_basic_test.py (revision 1401726) +++ system_test/replication_testsuite/replica_basic_test.py (working copy) @@ -185,53 +185,111 @@ # by "num_iterations" in testcase_n_properties.json # ============================================= i = 1 - numIterations = int(self.testcaseEnv.testcaseArgumentsDict["num_iteration"]) + numIterations = int(self.testcaseEnv.testcaseArgumentsDict["num_iteration"]) + brokerType = self.testcaseEnv.testcaseArgumentsDict["broker_type"] + bounceBrokerFlag = self.testcaseEnv.testcaseArgumentsDict["bounce_broker"] while i <= numIterations: - self.log_message("Iteration " + str(i) + " of " + str(numIterations)) - self.log_message("looking up leader") - leaderDict = kafka_system_test_utils.get_leader_elected_log_line( - self.systemTestEnv, self.testcaseEnv, self.leaderAttributesDict) + if brokerType == "leader" or brokerType == "follower": + self.log_message("looking up leader") + leaderDict = kafka_system_test_utils.get_leader_elected_log_line( + self.systemTestEnv, self.testcaseEnv, self.leaderAttributesDict) + + # ========================== + # leaderDict looks like this: + # ========================== + #{'entity_id': u'3', + # 'partition': '0', + # 'timestamp': 1345050255.8280001, + # 'hostname': u'localhost', + # 'topic': 'test_1', + # 'brokerid': '3'} + + if brokerType == "leader": + # validate to see if leader election is successful + self.log_message("validating leader election") + result = kafka_system_test_utils.validate_leader_election_successful( \ + self.testcaseEnv, leaderDict, self.testcaseEnv.validationStatusDict) + + # trigger leader re-election by stopping leader to get re-election latency + self.log_message("bounce_broker flag : " + bounceBrokerFlag) + if bounceBrokerFlag.lower() == "true": + reelectionLatency = kafka_system_test_utils.get_reelection_latency( + self.systemTestEnv, self.testcaseEnv, leaderDict, self.leaderAttributesDict) + latencyKeyName = "Leader Election Latency - iter " + str(i) + " brokerid " + leaderDict["brokerid"] + self.testcaseEnv.validationStatusDict[latencyKeyName] = str("{0:.2f}".format(reelectionLatency * 1000)) + " ms" + self.testcaseEnv.userDefinedEnvVarDict["leaderElectionLatencyList"].append("{0:.2f}".format(reelectionLatency * 1000)) + + time.sleep(1) + + # starting previously terminated broker + self.log_message("starting the previously terminated broker") + stoppedLeaderEntityId = leaderDict["entity_id"] + kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedLeaderEntityId) + else: + try: + # GC Pause simulation + hostname = leaderDict["hostname"] + pauseTime = self.testcaseEnv.testcaseArgumentsDict["pause_time_in_seconds"] + parentPid = self.testcaseEnv.entityBrokerParentPidDict[leaderDict["entity_id"]] + pidStack = system_test_utils.get_remote_child_processes(hostname, parentPid) + system_test_utils.simulate_garbage_collection_pause_in_remote_process(hostname, pidStack, pauseTime) + except: + pass + + else: # follower + # a list of all brokers + brokerEntityIdList = system_test_utils.get_data_from_list_of_dicts( + self.systemTestEnv.clusterEntityConfigDictList, "role", "broker", "entity_id") - # ========================== - # leaderDict looks like this: - # ========================== - #{'entity_id': u'3', - # 'partition': '0', - # 'timestamp': 1345050255.8280001, - # 'hostname': u'localhost', - # 'topic': 'test_1', - # 'brokerid': '3'} + # we pick the follower from the first broker which is not the leader + firstFollowerEntityId = None + for brokerEntityId in brokerEntityIdList: + if brokerEntityId != leaderDict["entity_id"]: + firstFollowerEntityId = brokerEntityId + break - # ============================================= - # validate to see if leader election is successful - # ============================================= - self.log_message("validating leader election") - result = kafka_system_test_utils.validate_leader_election_successful( \ - self.testcaseEnv, leaderDict, self.testcaseEnv.validationStatusDict) + # stopping Follower + self.log_message("bounce_broker flag : " + bounceBrokerFlag) + if bounceBrokerFlag.lower() == "true": + self.log_message("stopping follower with entity id: " + firstFollowerEntityId) + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, firstFollowerEntityId, + self.testcaseEnv.entityBrokerParentPidDict[firstFollowerEntityId]) + + time.sleep(1) + + # starting previously terminated broker + self.log_message("starting the previously terminated broker") + stoppedBrokerEntityId = firstFollowerEntityId + kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedBrokerEntityId) + + elif brokerType == "controller": + self.log_message("looking up controller") + controllerDict = kafka_system_test_utils.get_controller_attributes(self.systemTestEnv, self.testcaseEnv) - # ============================================= - # trigger leader re-election by stopping leader - # to get re-election latency - # ============================================= - bounceLeaderFlag = self.testcaseEnv.testcaseArgumentsDict["bounce_leader"] - self.log_message("bounce_leader flag : " + bounceLeaderFlag) - if (bounceLeaderFlag.lower() == "true"): - reelectionLatency = kafka_system_test_utils.get_reelection_latency( - self.systemTestEnv, self.testcaseEnv, leaderDict, self.leaderAttributesDict) - latencyKeyName = "Leader Election Latency - iter " + str(i) + " brokerid " + leaderDict["brokerid"] - self.testcaseEnv.validationStatusDict[latencyKeyName] = str("{0:.2f}".format(reelectionLatency * 1000)) + " ms" - self.testcaseEnv.userDefinedEnvVarDict["leaderElectionLatencyList"].append("{0:.2f}".format(reelectionLatency * 1000)) + # ========================== + # controllerDict looks like this: + # ========================== + #{'entity_id': u'3', + # 'timestamp': 1345050255.8280001, + # 'hostname': u'localhost', + # 'brokerid': '3'} + + # stopping Controller + self.log_message("bounce_broker flag : " + bounceBrokerFlag) + if bounceBrokerFlag.lower() == "true": + self.log_message("stopping controller : " + controllerDict["brokerid"]) + kafka_system_test_utils.stop_remote_entity(self.systemTestEnv, controllerDict["entity_id"], + self.testcaseEnv.entityBrokerParentPidDict[controllerDict["entity_id"]]) - # ============================================= - # starting previously terminated broker - # ============================================= - if bounceLeaderFlag.lower() == "true": - self.log_message("starting the previously terminated broker") - stoppedLeaderEntityId = leaderDict["entity_id"] - kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedLeaderEntityId) + time.sleep(1) + # starting previously terminated broker + self.log_message("starting the previously terminated broker") + stoppedBrokerEntityId = controllerDict["entity_id"] + kafka_system_test_utils.start_entity_in_background(self.systemTestEnv, self.testcaseEnv, stoppedBrokerEntityId) + self.anonLogger.info("sleeping for 15s") time.sleep(15) i += 1 @@ -279,8 +337,9 @@ # ============================================= # starting consumer # ============================================= - self.log_message("starting consumer in the background") - kafka_system_test_utils.start_console_consumer(self.systemTestEnv, self.testcaseEnv) + self.log_message("starting debug consumers in the background") + #kafka_system_test_utils.start_console_consumer(self.systemTestEnv, self.testcaseEnv) + kafka_system_test_utils.start_simple_consumer(self.systemTestEnv, self.testcaseEnv) self.anonLogger.info("sleeping for 10s") time.sleep(10) @@ -306,7 +365,8 @@ # validate the data matched and checksum # ============================================= self.log_message("validating data matched") - kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) + #kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv) + kafka_system_test_utils.validate_simple_consumer_data_matched(self.systemTestEnv, self.testcaseEnv) kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv) # ============================================= Index: system_test/replication_testsuite/testcase_0001/testcase_0001_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0001/testcase_0001_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0001/testcase_0001_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0002/testcase_0002_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0002/testcase_0002_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0002/testcase_0002_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0003/testcase_0003_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0003/testcase_0003_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0003/testcase_0003_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0004/testcase_0004_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0004/testcase_0004_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0004/testcase_0004_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0005/testcase_0005_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0005/testcase_0005_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0005/testcase_0005_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0006/testcase_0006_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0006/testcase_0006_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0006/testcase_0006_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0007/testcase_0007_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0007/testcase_0007_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0007/testcase_0007_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0008/testcase_0008_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0008/testcase_0008_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0008/testcase_0008_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0009/testcase_0009_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0009/testcase_0009_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0009/testcase_0009_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0010/testcase_0010_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0010/testcase_0010_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0010/testcase_0010_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 1048576 (1M)" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0021/testcase_0021_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0021/testcase_0021_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0021/testcase_0021_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0022/testcase_0022_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0022/testcase_0022_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0022/testcase_0022_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 512000" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0023/testcase_0023_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0023/testcase_0023_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0023/testcase_0023_properties.json (working copy) @@ -8,7 +8,8 @@ "07":"Log segment size : 512000" }, "testcase_args": { - "bounce_leader": "false", + "broker_type": "leader", + "bounce_broker": "false", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0101/testcase_0101_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0101/testcase_0101_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0101/testcase_0101_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0102/testcase_0102_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0102/testcase_0102_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0102/testcase_0102_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0103/testcase_0103_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0103/testcase_0103_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0103/testcase_0103_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0104/testcase_0104_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0104/testcase_0104_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0104/testcase_0104_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0105/testcase_0105_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0105/testcase_0105_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0105/testcase_0105_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "1", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0106/testcase_0106_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0106/testcase_0106_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0106/testcase_0106_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0107/testcase_0107_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0107/testcase_0107_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0107/testcase_0107_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0108/testcase_0108_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0108/testcase_0108_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0108/testcase_0108_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0109/testcase_0109_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0109/testcase_0109_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0109/testcase_0109_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0110/testcase_0110_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0110/testcase_0110_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0110/testcase_0110_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 1048576 (1M)" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "1", Index: system_test/replication_testsuite/testcase_0111/testcase_0111_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0111/testcase_0111_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0111/testcase_0111_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0112/testcase_0112_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0112/testcase_0112_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0112/testcase_0112_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0113/testcase_0113_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0113/testcase_0113_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0113/testcase_0113_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 102400" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 512000" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 512000" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "3", "num_iteration": "3", Index: system_test/replication_testsuite/testcase_0151/testcase_0151_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0151/testcase_0151_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0151/testcase_0151_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : Base Test", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0152/testcase_0152_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0152/testcase_0152_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0152/testcase_0152_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. mode => async", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0153/testcase_0153_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0153/testcase_0153_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0153/testcase_0153_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0154/testcase_0154_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0154/testcase_0154_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0154/testcase_0154_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. mode => async; 2. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0155/testcase_0155_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0155/testcase_0155_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0155/testcase_0155_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0156/testcase_0156_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0156/testcase_0156_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0156/testcase_0156_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. mode => async; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0157/testcase_0157_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0157/testcase_0157_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0157/testcase_0157_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. acks => 1; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0158/testcase_0158_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0158/testcase_0158_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0158/testcase_0158_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Leader Failures (SIGKILL) in Replication : 1. mode => async; 2. acks => 1; 3. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "true", + "signal_type": "SIGKILL", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0201/testcase_0201_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0201/testcase_0201_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0201/testcase_0201_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : Base Test", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0202/testcase_0202_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0202/testcase_0202_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0202/testcase_0202_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. mode => async", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0203/testcase_0203_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0203/testcase_0203_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0203/testcase_0203_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0204/testcase_0204_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0204/testcase_0204_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0204/testcase_0204_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. mode => async; 2. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0205/testcase_0205_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0205/testcase_0205_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0205/testcase_0205_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0206/testcase_0206_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0206/testcase_0206_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0206/testcase_0206_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. mode => async; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0207/testcase_0207_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0207/testcase_0207_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0207/testcase_0207_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. acks => 1; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0208/testcase_0208_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0208/testcase_0208_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0208/testcase_0208_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Controller Failures (SIGTERM) in Replication : 1. mode => async; 2. acks => 1; 3. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "controller", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0251/testcase_0251_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0251/testcase_0251_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0251/testcase_0251_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : Base Test", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0252/testcase_0252_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0252/testcase_0252_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0252/testcase_0252_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. mode => async", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0253/testcase_0253_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0253/testcase_0253_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0253/testcase_0253_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0254/testcase_0254_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0254/testcase_0254_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0254/testcase_0254_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. mode => async; 2. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:0", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0255/testcase_0255_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0255/testcase_0255_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0255/testcase_0255_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0256/testcase_0256_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0256/testcase_0256_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0256/testcase_0256_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. mode => async; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:-1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0257/testcase_0257_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0257/testcase_0257_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0257/testcase_0257_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. acks => 1; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:sync, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0258/testcase_0258_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0258/testcase_0258_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0258/testcase_0258_properties.json (revision 0) @@ -0,0 +1,81 @@ +{ + "description": {"01":"Multi Follower Failures (SIGTERM) in Replication : 1. mode => async; 2. acks => 1; 3. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)", + "05":"Restart the terminated broker", + "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully", + "07":"At the end it verifies the log size and contents", + "08":"Use a consumer to verify no message loss.", + "09":"Producer dimensions : mode:async, acks:1, comp:1", + "10":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "follower", + "bounce_broker": "true", + "signal_type": "SIGTERM", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0301/testcase_0301_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0301/testcase_0301_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0301/testcase_0301_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : Base Test", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0302/testcase_0302_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0302/testcase_0302_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0302/testcase_0302_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. mode => async", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0303/testcase_0303_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0303/testcase_0303_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0303/testcase_0303_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0304/testcase_0304_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0304/testcase_0304_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0304/testcase_0304_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. mode => async; 2. acks => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "0", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0305/testcase_0305_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0305/testcase_0305_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0305/testcase_0305_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0306/testcase_0306_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0306/testcase_0306_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0306/testcase_0306_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. mode => async; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "-1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0307/testcase_0307_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0307/testcase_0307_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0307/testcase_0307_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. acks => 1; 2. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"false", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_0308/testcase_0308_properties.json =================================================================== --- system_test/replication_testsuite/testcase_0308/testcase_0308_properties.json (revision 0) +++ system_test/replication_testsuite/testcase_0308/testcase_0308_properties.json (revision 0) @@ -0,0 +1,79 @@ +{ + "description": {"01":"Leader Garbage Collection Pauses Simulation in Replication : 1. mode => async; 2. acks => 1; 3. comp => 1", + "02":"Produce and consume messages to a single topic - three partition.", + "03":"This test sends messages to 3 replicas", + "04":"To simulate GC Pauses : kill -SIGSTOP => wait N seconds => kill -SIGCONT", + "05":"At the end it verifies the log size and contents", + "06":"Use a consumer to verify no message loss.", + "07":"Producer dimensions : mode:sync, acks:-1, comp:0", + "08":"Log segment size : 102400" + }, + "testcase_args": { + "broker_type": "leader", + "bounce_broker": "false", + "pause_time_in_seconds": "5", + "replica_factor": "3", + "num_partition": "3", + "num_iteration": "3", + "sleep_seconds_between_producer_calls": "1", + "message_producing_free_time_sec": "15", + "num_messages_to_produce_per_producer_call": "50" + }, + "entities": [ + { + "entity_id": "0", + "clientPort": "2188", + "dataDir": "/tmp/zookeeper_0", + "log_filename": "zookeeper_2188.log", + "config_filename": "zookeeper_2188.properties" + }, + { + "entity_id": "1", + "port": "9091", + "brokerid": "1", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_1_logs", + "log_filename": "kafka_server_9091.log", + "config_filename": "kafka_server_9091.properties" + }, + { + "entity_id": "2", + "port": "9092", + "brokerid": "2", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_2_logs", + "log_filename": "kafka_server_9092.log", + "config_filename": "kafka_server_9092.properties" + }, + { + "entity_id": "3", + "port": "9093", + "brokerid": "3", + "log.file.size": "102400", + "log.dir": "/tmp/kafka_server_3_logs", + "log_filename": "kafka_server_9093.log", + "config_filename": "kafka_server_9093.properties" + }, + { + "entity_id": "4", + "topic": "test_1", + "threads": "5", + "compression-codec": "1", + "message-size": "500", + "message": "100", + "request-num-acks": "1", + "async":"true", + "log_filename": "producer_performance.log", + "config_filename": "producer_performance.properties" + }, + { + "entity_id": "5", + "topic": "test_1", + "groupid": "mytestgroup", + "consumer-timeout-ms": "10000", + "zookeeper": "localhost:2188", + "log_filename": "console_consumer.log", + "config_filename": "console_consumer.properties" + } + ] +} Index: system_test/replication_testsuite/testcase_1/testcase_1_properties.json =================================================================== --- system_test/replication_testsuite/testcase_1/testcase_1_properties.json (revision 1401726) +++ system_test/replication_testsuite/testcase_1/testcase_1_properties.json (working copy) @@ -11,7 +11,8 @@ "10":"Log segment size : 10240" }, "testcase_args": { - "bounce_leader": "true", + "broker_type": "leader", + "bounce_broker": "true", "replica_factor": "3", "num_partition": "2", "num_iteration": "2", Index: system_test/system_test_runner.py =================================================================== --- system_test/system_test_runner.py (revision 1401726) +++ system_test/system_test_runner.py (working copy) @@ -166,6 +166,7 @@ print if not systemTestEnv.printTestDescriptionsOnly: + totalFailureCount = 0 print print "========================================================" print " TEST REPORTS" @@ -174,14 +175,26 @@ for key in sorted(systemTestResult.iterkeys()): if key == "validation_status": print key, " : " + testItemStatus = None for validatedItem in sorted(systemTestResult[key].iterkeys()): - print " ", validatedItem, " : ", systemTestResult[key][validatedItem] + testItemStatus = systemTestResult[key][validatedItem] + print " ", validatedItem, " : ", testItemStatus + if "FAILED" == testItemStatus: + totalFailureCount += 1 else: print key, " : ", systemTestResult[key] print print "========================================================" print + print "========================================================" + print "Total failures count : " + str(totalFailureCount) + print "========================================================" + print + return totalFailureCount + + return -1 + # ========================= # main entry point # ========================= Index: system_test/testcase_to_run_all.json =================================================================== --- system_test/testcase_to_run_all.json (revision 0) +++ system_test/testcase_to_run_all.json (revision 0) @@ -0,0 +1,87 @@ +{ + "ReplicaBasicTest" : [ + "testcase_0001", + "testcase_0002", + "testcase_0003", + "testcase_0004", + "testcase_0005", + "testcase_0006", + "testcase_0007", + "testcase_0008", + "testcase_0009", + "testcase_0010", + + "testcase_0021", + "testcase_0022", + "testcase_0023", + + "testcase_0101", + "testcase_0102", + "testcase_0103", + "testcase_0104", + "testcase_0105", + "testcase_0106", + "testcase_0107", + "testcase_0108", + "testcase_0109", + "testcase_0110", + + "testcase_0111", + "testcase_0112", + "testcase_0113", + "testcase_0114", + "testcase_0115", + "testcase_0116", + "testcase_0117", + "testcase_0118", + + "testcase_0121", + "testcase_0122", + "testcase_0123", + + "testcase_0151", + "testcase_0152", + "testcase_0153", + "testcase_0154", + "testcase_0155", + "testcase_0156", + "testcase_0157", + "testcase_0158", + + "testcase_0201", + "testcase_0202", + "testcase_0203", + "testcase_0204", + "testcase_0205", + "testcase_0206", + "testcase_0207", + "testcase_0208", + + "testcase_0251", + "testcase_0252", + "testcase_0253", + "testcase_0254", + "testcase_0255", + "testcase_0256", + "testcase_0257", + "testcase_0258", + + "testcase_0301", + "testcase_0302", + "testcase_0303", + "testcase_0304", + "testcase_0305", + "testcase_0306", + "testcase_0307", + "testcase_0308" + ], + + "MigrationToolTest" : [ + "testcase_9001" + ], + + "MirrorMakerTest" : [ + "testcase_5001", + "testcase_5002" + ] +} Index: system_test/utils/kafka_system_test_utils.py =================================================================== --- system_test/utils/kafka_system_test_utils.py (revision 1401726) +++ system_test/utils/kafka_system_test_utils.py (working copy) @@ -22,6 +22,7 @@ import datetime import getpass +import hashlib import inspect import json import logging @@ -80,6 +81,8 @@ # type is either "metrics" or "dashboards" or "default" if type == "metrics": return testcaseEnv.testCaseLogsDir + "/" + role + "-" + entityId + "/metrics" + elif type == "log_segments" : + return testcaseEnv.testCaseLogsDir + "/" + role + "-" + entityId + "/log_segments" elif type == "default" : return testcaseEnv.testCaseLogsDir + "/" + role + "-" + entityId elif type == "dashboards": @@ -110,19 +113,19 @@ metricsPathName = get_testcase_config_log_dir_pathname(testcaseEnv, role, entityId, "metrics") if not os.path.exists(metricsPathName) : os.makedirs(metricsPathName) - + # create the role directory under dashboards dashboardsRoleDir = dashboardsPathName + "/" + role if not os.path.exists(dashboardsRoleDir) : os.makedirs(dashboardsRoleDir) - def collect_logs_from_remote_hosts(systemTestEnv, testcaseEnv): anonLogger.info("================================================") anonLogger.info("collecting logs from remote machines") anonLogger.info("================================================") testCaseBaseDir = testcaseEnv.testCaseBaseDir + tcConfigsList = testcaseEnv.testcaseConfigsList for clusterEntityConfigDict in systemTestEnv.clusterEntityConfigDictList: hostname = clusterEntityConfigDict["hostname"] @@ -157,6 +160,20 @@ logger.debug("executing command [" + cmdStr + "]", extra=d) system_test_utils.sys_call(cmdStr) + # ============================== + # collect broker log segment file + # ============================== + if role == "broker": + dataLogPathName = system_test_utils.get_data_by_lookup_keyval( + testcaseEnv.testcaseConfigsList, "entity_id", entity_id, "log.dir") + + cmdList = ["scp -r", + hostname + ":" + dataLogPathName, + logPathName] + cmdStr = " ".join(cmdList) + logger.debug("executing command [" + cmdStr + "]", extra=d) + system_test_utils.sys_call(cmdStr) + # ============================== # collect dashboards file # ============================== @@ -321,7 +338,8 @@ testcaseEnv.userDefinedEnvVarDict["targetZkHostPortDict"][key] = val else: - logger.error("Unknown cluster name: " + clusterName) + logger.error("Invalid cluster name: " + clusterName, extra=d) + raise Exception("Invalid cluster name : " + clusterName) sys.exit(1) # update broker cluster info into "testcaseEnv.userDefinedEnvVarDict" @@ -344,7 +362,8 @@ else: testcaseEnv.userDefinedEnvVarDict["targetBrokerList"] += "," + hostname + ":" + port else: - logger.error("Unknown cluster name: " + clusterName) + logger.error("Invalid cluster name: " + clusterName, extra=d) + raise Exception("Invalid cluster name : " + clusterName) sys.exit(1) # for each entity in the cluster config @@ -363,7 +382,7 @@ elif clusterCfg["cluster_name"] == "target": tcCfg["zk.connect"] = testcaseEnv.userDefinedEnvVarDict["targetZkConnectStr"] else: - logger.error("Unknown cluster name: " + clusterName) + logger.error("Unknown cluster name: " + clusterName, extra=d) sys.exit(1) addedCSVConfig = {} @@ -384,7 +403,7 @@ cfgDestPathname + "/" + tcCfg["config_filename"], tcCfg, testcaseEnv.userDefinedEnvVarDict["targetZkHostPortDict"]) else: - logger.error("Unknown cluster name: " + clusterName) + logger.error("Unknown cluster name: " + clusterName, extra=d) sys.exit(1) elif ( clusterCfg["role"] == "mirror_maker"): @@ -397,32 +416,6 @@ copy_file_with_dict_values(cfgTemplatePathname + "/mirror_consumer.properties", cfgDestPathname + "/" + tcCfg["mirror_consumer_config_filename"], tcCfg, None) - elif ( clusterCfg["role"] == "producer_performance" ): - copy_file_with_dict_values(cfgTemplatePathname + "/producer_performance.properties", - cfgDestPathname + "/" + tcCfg["config_filename"], tcCfg, None) - - elif ( clusterCfg["role"] == "console_consumer" ): - copy_file_with_dict_values(cfgTemplatePathname + "/console_consumer.properties", - cfgDestPathname + "/" + tcCfg["config_filename"], tcCfg, None) - - elif ( clusterCfg["role"] == "producer" ): - addedCSVConfig = {} - addedCSVConfig["kafka.csv.metrics.dir"] = get_testcase_config_log_dir_pathname(testcaseEnv, "producer", clusterCfg["entity_id"], "metrics") - addedCSVConfig["kafka.metrics.polling.interval.secs"] = "5" - addedCSVConfig["kafka.metrics.reporters"] = "kafka.metrics.KafkaCSVMetricsReporter" - addedCSVConfig["kfka.metrics.polling.interval.secsafka.csv.metrics.reporter.enabled"] = "true" - copy_file_with_dict_values(cfgTemplatePathname + "/producer.properties", - cfgDestPathname + "/" + tcCfg["config_filename"], tcCfg, addedCSVConfig) - - - elif ( clusterCfg["role"] == "consumer" ): - addedCSVConfig = {} - addedCSVConfig["kafka.csv.metrics.dir"] = get_testcase_config_log_dir_pathname(testcaseEnv, "consumer", clusterCfg["entity_id"], "metrics") - addedCSVConfig["kafka.metrics.polling.interval.secs"] = "5" - addedCSVConfig["kafka.metrics.reporters"] = "kafka.metrics.KafkaCSVMetricsReporter" - addedCSVConfig["kfka.metrics.polling.interval.secsafka.csv.metrics.reporter.enabled"] = "true" - copy_file_with_dict_values(cfgTemplatePathname + "/consumer.properties", - cfgDestPathname + "/" + tcCfg["config_filename"], tcCfg, addedCSVConfig) else: logger.debug("UNHANDLED role " + clusterCfg["role"], extra=d) @@ -747,7 +740,7 @@ elif clusterName == "target": zkConnectStr = testcaseEnv.userDefinedEnvVarDict["targetZkConnectStr"] else: - logger.error("Invalid cluster name : " + clusterName) + logger.error("Invalid cluster name : " + clusterName, extra=d) sys.exit(1) cmdList = ["ssh " + host, @@ -836,7 +829,7 @@ elif clusterName == "target": brokerListStr = testcaseEnv.userDefinedEnvVarDict["targetBrokerList"] else: - logger.error("Unknown cluster name: " + clusterName) + logger.error("Unknown cluster name: " + clusterName, extra=d) sys.exit(1) logger.info("starting producer preformance", extra=d) @@ -942,16 +935,23 @@ testcaseEnv.lock.release() time.sleep(1) -def stop_remote_entity(systemTestEnv, entityId, parentPid): +def stop_remote_entity(systemTestEnv, entityId, parentPid, signalType="SIGTERM"): clusterEntityConfigDictList = systemTestEnv.clusterEntityConfigDictList hostname = system_test_utils.get_data_by_lookup_keyval(clusterEntityConfigDictList, "entity_id", entityId, "hostname") pidStack = system_test_utils.get_remote_child_processes(hostname, parentPid) - logger.debug("terminating process id: " + parentPid + " in host: " + hostname, extra=d) - system_test_utils.sigterm_remote_process(hostname, pidStack) + logger.debug("terminating (" + signalType + ") process id: " + parentPid + " in host: " + hostname, extra=d) + if signalType.lower() == "sigterm": + system_test_utils.sigterm_remote_process(hostname, pidStack) + elif signalType.lower() == "sigkill": + system_test_utils.sigkill_remote_process(hostname, pidStack) + else: + logger.error("Invalid signal type: " + signalType, extra=d) + raise Exception("Invalid signal type: " + signalType) + def force_stop_remote_entity(systemTestEnv, entityId, parentPid): clusterEntityConfigDictList = systemTestEnv.clusterEntityConfigDictList @@ -1048,6 +1048,7 @@ for prodPerfCfg in prodPerfCfgList: producerEntityId = prodPerfCfg["entity_id"] topic = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", producerEntityId, "topic") + acks = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", producerEntityId, "request-num-acks") consumerEntityIdList = system_test_utils.get_data_from_list_of_dicts( \ clusterEntityConfigDictList, "role", "console_consumer", "entity_id") @@ -1089,11 +1090,15 @@ if ( len(missingMsgIdInConsumer) == 0 and len(producerMsgIdSet) > 0 ): validationStatusDict["Validate for data matched on topic [" + topic + "]"] = "PASSED" - #return True + elif (acks == "1"): + missingPercentage = len(missingMsgIdInConsumer) * 100 / len(producerMsgIdSet) + print "#### missing Percent : ", missingPercentage + if missingPercentage <= 1: + validationStatusDict["Validate for data matched on topic [" + topic + "]"] = "PASSED" + logger.warn("Test case passes with less than 1% data loss : [" + str(len(missingMsgIdInConsumer)) + "] missing messages", extra=d) else: validationStatusDict["Validate for data matched on topic [" + topic + "]"] = "FAILED" logger.info("See " + msgIdMissingInConsumerLogPathName + " for missing MessageID", extra=d) - #return False def validate_leader_election_successful(testcaseEnv, leaderDict, validationStatusDict): @@ -1215,6 +1220,7 @@ leaderBrokerId = None leaderPPid = None shutdownLeaderTimestamp = None + leaderReElectionLatency = -1 if testcaseEnv.validationStatusDict["Validate leader election successful"] == "FAILED": # leader election is not successful - something is wrong => so skip this testcase @@ -1230,132 +1236,44 @@ raise logger.info("stopping leader in entity "+leaderEntityId+" with pid "+leaderPPid, extra=d) - stop_remote_entity(systemTestEnv, leaderEntityId, leaderPPid) + signalType = None + try: + signalType = testcaseEnv.testcaseArgumentsDict["signal_type"] + except: + pass + if signalType is None or signalType.lower() == "sigterm": + stop_remote_entity(systemTestEnv, leaderEntityId, leaderPPid) + elif signalType.lower() == "sigkill": + stop_remote_entity(systemTestEnv, leaderEntityId, leaderPPid, "SIGKILL") + else: + logger.error("Unsupported signal type: " + signalType, extra=d) + raise Exception("Unsupported signal type: " + signalType) + logger.info("sleeping for 10s for leader re-election to complete", extra=d) time.sleep(10) # get broker shut down completed timestamp shutdownBrokerDict = get_broker_shutdown_log_line(systemTestEnv, testcaseEnv, leaderAttributesDict) - logger.debug("unix timestamp of shut down completed: " + str("{0:.6f}".format(shutdownBrokerDict["timestamp"])), extra=d) + shutdownTimestamp = -1 + try: + shutdownTimestamp = shutdownBrokerDict["timestamp"] + logger.debug("unix timestamp of shut down completed: " + str("{0:.6f}".format(shutdownTimestamp)), extra=d) + except: + logger.warn("unable to find broker shut down timestamp", extra=d) + logger.info("looking up new leader", extra=d) - leaderDict2 = get_leader_elected_log_line(systemTestEnv, testcaseEnv, leaderAttributesDict) logger.debug("unix timestamp of new elected leader: " + str("{0:.6f}".format(leaderDict2["timestamp"])), extra=d) - leaderReElectionLatency = float(leaderDict2["timestamp"]) - float(shutdownBrokerDict["timestamp"]) - logger.info("leader Re-election Latency: " + str(leaderReElectionLatency) + " sec", extra=d) + if shutdownTimestamp > 0: + leaderReElectionLatency = float(leaderDict2["timestamp"]) - float(shutdownTimestamp) + logger.info("leader Re-election Latency: " + str(leaderReElectionLatency) + " sec", extra=d) return leaderReElectionLatency -def validate_broker_log_segment_checksum(systemTestEnv, testcaseEnv): - anonLogger.info("================================================") - anonLogger.info("validating broker log segment checksums") - anonLogger.info("================================================") - - # brokerLogCksumDict - - # a dictionary to keep track of log segment files in each brokers and will look like this: - # - # {u'broker-1': {'test_1-0/00000000000000000000.kafka': '91500855', - # 'test_1-0/00000000000000010255.kafka': '1906285795', - # 'test_1-1/00000000000000000000.kafka': '3785861722', - # 'test_1-1/00000000000000010322.kafka': '1731628308'}, - # u'broker-2': {'test_1-0/00000000000000000000.kafka': '91500855', - # 'test_1-0/00000000000000010255.kafka': '1906285795', - # 'test_1-1/00000000000000000000.kafka': '3785861722', - # 'test_1-1/00000000000000010322.kafka': '1731628308'}, - # u'broker-3': {'test_1-0/00000000000000000000.kafka': '91500855', - # 'test_1-0/00000000000000010255.kafka': '1906285795', - # 'test_1-1/00000000000000000000.kafka': '3431356313'}} - brokerLogCksumDict = {} - - clusterEntityConfigDictList = systemTestEnv.clusterEntityConfigDictList - brokerEntityIdList = system_test_utils.get_data_from_list_of_dicts(clusterEntityConfigDictList, "role", "broker", "entity_id") - - # access all brokers' hosts to get broker id and its corresponding log - # segment file checksums and populate brokerLogCksumDict - for brokerEntityId in brokerEntityIdList: - logCksumDict = {} - - hostname = system_test_utils.get_data_by_lookup_keyval(clusterEntityConfigDictList, "entity_id", brokerEntityId, "hostname") - logDir = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", brokerEntityId, "log.dir") - - # get the log segment file full path name - cmdStr = "ssh " + hostname + " \"find " + logDir + " -name '*.log'\" 2> /dev/null" - logger.debug("executing command [" + cmdStr + "]", extra=d) - subproc = system_test_utils.sys_call_return_subproc(cmdStr) - for line in subproc.stdout.readlines(): - # Need a key to identify each corresponding log segment file in different brokers: - # This can be achieved by using part of the full log segment file path as a key to identify - # the individual log segment file checksums. The key can be extracted from the path name - # and starting from "topic-partition" such as: - # full log segment path name : /tmp/kafka_server_1_logs/test_1-0/00000000000000010255.kafka - # part of the path name as key : test_1-0/00000000000000010255.kafka - logSegmentPathName = line.rstrip('\n') - substrIndex = logSegmentPathName.index(logDir) + len(logDir + "/") - logSegmentFile = logSegmentPathName[substrIndex:] - - # get log segment file checksum - cksumCmdStr = "ssh " + hostname + " \"cksum " + logSegmentPathName + " | cut -f1 -d ' '\" 2> /dev/null" - subproc2 = system_test_utils.sys_call_return_subproc(cksumCmdStr) - for line2 in subproc2.stdout.readlines(): - checksum = line2.rstrip('\n') - # use logSegmentFile as a key to associate with its checksum - logCksumDict[logSegmentFile] = checksum - - # associate this logCksumDict with its broker id - brokerLogCksumDict["broker-"+brokerEntityId] = logCksumDict - - # use a list of sets for checksums comparison - sets = [] - for brokerId, logCksumDict in brokerLogCksumDict.items(): - sets.append( set(logCksumDict.items()) ) - - # looping through the sets and compare between broker[n] & broker[n+1] ... - idx = 0 - diffItemSet = None - while idx < len(sets) - 1: - diffItemSet = sets[idx] ^ sets[idx + 1] - - if (len(diffItemSet) > 0): - logger.error("Mismatch found : " + str(diffItemSet), extra=d) - testcaseEnv.validationStatusDict["Log segment checksum matching across all replicas"] = "FAILED" - - # get the mismatched items key, i.e. the log segment file name - diffItemList = list(diffItemSet) - diffItemKeys = [] - for keyvalSet in diffItemList: - keyvalList = list(keyvalSet) - diffItemKeys.append(keyvalList[0]) - - # mismatch found - so print out the whole log segment file checksum - # info with the mismatched checksum highlighted - for brokerId in sorted(brokerLogCksumDict.iterkeys()): - logCksumDict = brokerLogCksumDict[brokerId] - print brokerId,":" - for logSegmentFile in sorted(logCksumDict.iterkeys()): - checksum = logCksumDict[logSegmentFile] - sys.stdout.write(logSegmentFile + " => " + checksum) - try: - if diffItemKeys.index(logSegmentFile) >= 0: - sys.stdout.write(" <<<< not matching across all replicas") - except: - pass - print - print - return - idx += 1 - - # getting here means all log segment checksums matched - testcaseEnv.validationStatusDict["Log segment checksum matching across all replicas"] = "PASSED" - - anonLogger.info("log segment files checksum :") - print - pprint.pprint(brokerLogCksumDict) - print - def stop_all_remote_running_processes(systemTestEnv, testcaseEnv): entityConfigs = systemTestEnv.clusterEntityConfigDictList @@ -1488,4 +1406,320 @@ logger.info("See " + msgChecksumMissingInConsumerLogPathName + " for missing MessageID", extra=d) #return False +def validate_broker_log_segment_checksum(systemTestEnv, testcaseEnv, clusterName="source"): + anonLogger.info("================================================") + anonLogger.info("validating merged broker log segment checksums") + anonLogger.info("================================================") + + brokerLogCksumDict = {} + testCaseBaseDir = testcaseEnv.testCaseBaseDir + tcConfigsList = testcaseEnv.testcaseConfigsList + validationStatusDict = testcaseEnv.validationStatusDict + clusterConfigList = systemTestEnv.clusterEntityConfigDictList + #brokerEntityIdList = system_test_utils.get_data_from_list_of_dicts(clusterConfigList, "role", "broker", "entity_id") + allBrokerConfigList = system_test_utils.get_dict_from_list_of_dicts(clusterConfigList, "role", "broker") + brokerEntityIdList = system_test_utils.get_data_from_list_of_dicts(allBrokerConfigList, "cluster_name", clusterName, "entity_id") + + # loop through all brokers + for brokerEntityId in brokerEntityIdList: + logCksumDict = {} + # remoteLogSegmentPathName : /tmp/kafka_server_4_logs + # => remoteLogSegmentDir : kafka_server_4_logs + remoteLogSegmentPathName = system_test_utils.get_data_by_lookup_keyval(tcConfigsList, "entity_id", brokerEntityId, "log.dir") + remoteLogSegmentDir = os.path.basename(remoteLogSegmentPathName) + logPathName = get_testcase_config_log_dir_pathname(testcaseEnv, "broker", brokerEntityId, "default") + localLogSegmentPath = logPathName + "/" + remoteLogSegmentDir + + # localLogSegmentPath : + # .../system_test/mirror_maker_testsuite/testcase_5002/logs/broker-4/kafka_server_4_logs + # |- test_1-0 + # |- 00000000000000000000.index + # |- 00000000000000000000.log + # |- 00000000000000000020.index + # |- 00000000000000000020.log + # |- . . . + # |- test_1-1 + # |- 00000000000000000000.index + # |- 00000000000000000000.log + # |- 00000000000000000020.index + # |- 00000000000000000020.log + # |- . . . + + # loop through all topicPartition directories such as : test_1-0, test_1-1, ... + for topicPartition in os.listdir(localLogSegmentPath): + # found a topic-partition directory + if os.path.isdir(localLogSegmentPath + "/" + topicPartition): + # md5 hasher + m = hashlib.md5() + + # logSegmentKey is like this : kafka_server_9_logs:test_1-0 (delimited by ':') + logSegmentKey = remoteLogSegmentDir + ":" + topicPartition + + # log segment files are located in : localLogSegmentPath + "/" + topicPartition + # sort the log segment files under each topic-partition and get the md5 checksum + for logFile in sorted(os.listdir(localLogSegmentPath + "/" + topicPartition)): + # only process log file: *.log + if logFile.endswith(".log"): + # read the log segment file as binary + offsetLogSegmentPathName = localLogSegmentPath + "/" + topicPartition + "/" + logFile + fin = file(offsetLogSegmentPathName, 'rb') + # keep reading 64K max at a time + while True: + data = fin.read(65536) + if not data: + fin.close() + break + # update it into the hasher + m.update(data) + + # update the md5 checksum into brokerLogCksumDict with the corresponding key + brokerLogCksumDict[logSegmentKey] = m.hexdigest() + + # print it out to the console for reference + pprint.pprint(brokerLogCksumDict) + + # brokerLogCksumDict will look like this: + # { + # 'kafka_server_1_logs:tests_1-0': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_1_logs:tests_1-1': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_1_logs:tests_2-0': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_1_logs:tests_2-1': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_2_logs:tests_1-0': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_2_logs:tests_1-1': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_2_logs:tests_2-0': 'd41d8cd98f00b204e9800998ecf8427e', + # 'kafka_server_2_logs:tests_2-1': 'd41d8cd98f00b204e9800998ecf8427e' + # } + + checksumDict = {} + # organize the checksum according to their topic-partition and checksumDict will look like this: + # { + # 'test_1-0' : ['d41d8cd98f00b204e9800998ecf8427e','d41d8cd98f00b204e9800998ecf8427e'], + # 'test_1-1' : ['d41d8cd98f00b204e9800998ecf8427e','d41d8cd98f00b204e9800998ecf8427e'], + # 'test_2-0' : ['d41d8cd98f00b204e9800998ecf8427e','d41d8cd98f00b204e9800998ecf8427e'], + # 'test_2-1' : ['d41d8cd98f00b204e9800998ecf8427e','d41d8cd98f00b204e9800998ecf8427e'] + # } + + for brokerTopicPartitionKey, md5Checksum in brokerLogCksumDict.items(): + tokens = brokerTopicPartitionKey.split(":") + brokerKey = tokens[0] + topicPartition = tokens[1] + if topicPartition in checksumDict: + # key already exist + checksumDict[topicPartition].append(md5Checksum) + else: + # new key => create a new list to store checksum + checksumDict[topicPartition] = [] + checksumDict[topicPartition].append(md5Checksum) + + failureCount = 0 + + # loop through checksumDict: the checksums should be the same inside each + # topic-partition's list. Otherwise, checksum mismatched is detected + for topicPartition, checksumList in checksumDict.items(): + checksumSet = frozenset(checksumList) + if len(checksumSet) > 1: + failureCount += 1 + logger.error("merged log segment checksum in " + topicPartition + " mismatched", extra=d) + elif len(checksumSet) == 1: + logger.debug("merged log segment checksum in " + topicPartition + " matched", extra=d) + else: + logger.error("unexpected error in " + topicPartition, extra=d) + + if failureCount == 0: + validationStatusDict["Validate for merged log segment checksum in cluster [" + clusterName + "]"] = "PASSED" + else: + validationStatusDict["Validate for merged log segment checksum in cluster [" + clusterName + "]"] = "FAILED" + +def start_simple_consumer(systemTestEnv, testcaseEnv): + + clusterList = systemTestEnv.clusterEntityConfigDictList + consumerConfigList = system_test_utils.get_dict_from_list_of_dicts(clusterList, "role", "console_consumer") + for consumerConfig in consumerConfigList: + host = consumerConfig["hostname"] + entityId = consumerConfig["entity_id"] + jmxPort = consumerConfig["jmx_port"] + clusterName = consumerConfig["cluster_name"] + kafkaHome = system_test_utils.get_data_by_lookup_keyval(clusterList, "entity_id", entityId, "kafka_home") + javaHome = system_test_utils.get_data_by_lookup_keyval(clusterList, "entity_id", entityId, "java_home") + kafkaRunClassBin = kafkaHome + "/bin/kafka-run-class.sh" + consumerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", entityId, "default") + + # testcase configurations: + testcaseList = testcaseEnv.testcaseConfigsList + topic = system_test_utils.get_data_by_lookup_keyval(testcaseList, "entity_id", entityId, "topic") + + brokerListStr = "" + if clusterName == "source": + brokerListStr = testcaseEnv.userDefinedEnvVarDict["sourceBrokerList"] + elif clusterName == "target": + brokerListStr = testcaseEnv.userDefinedEnvVarDict["targetBrokerList"] + else: + logger.error("Invalid cluster name : " + clusterName, extra=d) + raise Exception("Invalid cluster name : " + clusterName) + + if len(brokerListStr) == 0: + logger.error("Empty broker list str", extra=d) + raise Exception("Empty broker list str") + + numPartitions = None + try: + numPartitions = testcaseEnv.testcaseArgumentsDict["num_partition"] + except: + pass + + if numPartitions is None: + logger.error("Invalid no. of partitions: " + numPartitions, extra=d) + raise Exception("Invalid no. of partitions: " + numPartitions) + else: + numPartitions = int(numPartitions) + + replicaIndex = 1 + brokerPortList = brokerListStr.split(',') + for brokerPort in brokerPortList: + + k = 0 + while (k < numPartitions): + logger.info("starting debug consumer for replica on [" + brokerPort + "] partition [" + str(k) + "]", extra=d) + brokerPortLabel = brokerPort.replace(":", "_") + cmdList = ["ssh " + host, + "'JAVA_HOME=" + javaHome, + kafkaRunClassBin + " kafka.tools.SimpleConsumerShell", + "--broker-list " + brokerListStr, + "--topic " + topic, + "--partition " + str(k), + "--replica " + str(replicaIndex), + "--no-wait-at-logend ", + " >> " + consumerLogPath + "/simple_consumer_" + str(replicaIndex) + ".log", + " & echo pid:$! > " + consumerLogPath + "/entity_" + entityId + "_pid'"] + + cmdStr = " ".join(cmdList) + + logger.debug("executing command: [" + cmdStr + "]", extra=d) + system_test_utils.async_sys_call(cmdStr) + time.sleep(2) + + pidCmdStr = "ssh " + host + " 'cat " + consumerLogPath + "/entity_" + entityId + "_pid'" + logger.debug("executing command: [" + pidCmdStr + "]", extra=d) + subproc = system_test_utils.sys_call_return_subproc(pidCmdStr) + + # keep track of the remote entity pid in a dictionary + for line in subproc.stdout.readlines(): + if line.startswith("pid"): + line = line.rstrip('\n') + logger.debug("found pid line: [" + line + "]", extra=d) + tokens = line.split(':') + testcaseEnv.consumerHostParentPidDict[host] = tokens[1] + + logger.info("sleeping for 5 sec",extra=d) + time.sleep(5) + k += 1 + replicaIndex += 1 + +def validate_simple_consumer_data_matched(systemTestEnv, testcaseEnv): + validationStatusDict = testcaseEnv.validationStatusDict + clusterEntityConfigDictList = systemTestEnv.clusterEntityConfigDictList + + prodPerfCfgList = system_test_utils.get_dict_from_list_of_dicts(clusterEntityConfigDictList, "role", "producer_performance") + consumerCfgList = system_test_utils.get_dict_from_list_of_dicts(clusterEntityConfigDictList, "role", "console_consumer") + + mismatchCount = 0 + + for prodPerfCfg in prodPerfCfgList: + producerEntityId = prodPerfCfg["entity_id"] + topic = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", producerEntityId, "topic") + acks = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", producerEntityId, "request-num-acks") + logger.debug("request-num-acks [" + acks + "]", extra=d) + + consumerEntityIdList = system_test_utils.get_data_from_list_of_dicts( \ + clusterEntityConfigDictList, "role", "console_consumer", "entity_id") + + matchingConsumerEntityId = None + for consumerEntityId in consumerEntityIdList: + consumerTopic = system_test_utils.get_data_by_lookup_keyval(testcaseEnv.testcaseConfigsList, "entity_id", consumerEntityId, "topic") + if consumerTopic in topic: + matchingConsumerEntityId = consumerEntityId + break + + if matchingConsumerEntityId is None: + break + + producerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "producer_performance", producerEntityId, "default") + producerLogPathName = producerLogPath + "/producer_performance.log" + producerMsgIdList = get_message_id(producerLogPathName) + producerMsgIdSet = set(producerMsgIdList) + logger.info("no. of unique messages on topic [" + topic + "] sent from publisher : " + str(len(producerMsgIdSet)), extra=d) + validationStatusDict["Unique messages from producer on [" + topic + "]"] = str(len(producerMsgIdSet)) + + consumerLogPath = get_testcase_config_log_dir_pathname(testcaseEnv, "console_consumer", matchingConsumerEntityId, "default") + for logFile in sorted(os.listdir(consumerLogPath)): + # only process log file: *.log + if logFile.endswith(".log"): + consumerLogPathName = consumerLogPath + "/" + logFile + consumerMsgIdList = get_message_id(consumerLogPathName) + consumerMsgIdSet = set(consumerMsgIdList) + missingMsgIdInConsumer = producerMsgIdSet - consumerMsgIdSet + msgIdMissingInConsumerLogPathName = get_testcase_config_log_dir_pathname( + testcaseEnv, "console_consumer", matchingConsumerEntityId, "default") + \ + "/" + logFile + "_msg_id_missing_in_consumer.log" + + outfile = open(msgIdMissingInConsumerLogPathName, "w") + for id in missingMsgIdInConsumer: + outfile.write(id + "\n") + outfile.close() + + logger.info("no. of unique messages on topic [" + topic + "] at " + logFile + " : " + str(len(consumerMsgIdSet)), extra=d) + validationStatusDict["Unique messages from consumer on [" + topic + "] at " + logFile] = str(len(consumerMsgIdSet)) + + if acks == "-1" and len(missingMsgIdInConsumer) > 0: + mismatchCount += 1 + elif acks == "1" and len(missingMsgIdInConsumer) > 0: + missingPercentage = len(missingMsgIdInConsumer) * 100 / len(producerMsgIdSet) + logger.debug("missing percentage [" + str(missingPercentage) + "]", extra=d) + if missingPercentage <= 1: + logger.warn("Test case (acks == 1) passes with < 1% data loss : [" + \ + str(len(missingMsgIdInConsumer)) + "] missing messages", extra=d) + else: + mismatchCount += 1 + + if mismatchCount == 0: + validationStatusDict["Validate for data matched on topic [" + topic + "]"] = "PASSED" + else: + validationStatusDict["Validate for data matched on topic [" + topic + "]"] = "FAILED" + +def get_controller_attributes(systemTestEnv, testcaseEnv): + + logger.info("Querying Zookeeper for Controller info ...", extra=d) + + # keep track of controller data in this dict such as broker id & entity id + controllerDict = {} + + clusterConfigsList = systemTestEnv.clusterEntityConfigDictList + tcConfigsList = testcaseEnv.testcaseConfigsList + + zkDictList = system_test_utils.get_dict_from_list_of_dicts(clusterConfigsList, "role", "zookeeper") + firstZkDict = zkDictList[0] + hostname = firstZkDict["hostname"] + zkEntityId = firstZkDict["entity_id"] + clientPort = system_test_utils.get_data_by_lookup_keyval(tcConfigsList, "entity_id", zkEntityId, "clientPort") + kafkaHome = system_test_utils.get_data_by_lookup_keyval(clusterConfigsList, "entity_id", zkEntityId, "kafka_home") + javaHome = system_test_utils.get_data_by_lookup_keyval(clusterConfigsList, "entity_id", zkEntityId, "java_home") + kafkaRunClassBin = kafkaHome + "/bin/kafka-run-class.sh" + + cmdStrList = ["ssh " + hostname, + "\"JAVA_HOME=" + javaHome, + kafkaRunClassBin + " org.apache.zookeeper.ZooKeeperMain", + "-server " + testcaseEnv.userDefinedEnvVarDict["sourceZkConnectStr"], + "'get /controller' 2> /dev/null | tail -1\""] + + cmdStr = " ".join(cmdStrList) + logger.debug("executing command [" + cmdStr + "]", extra=d) + subproc = system_test_utils.sys_call_return_subproc(cmdStr) + for line in subproc.stdout.readlines(): + brokerid = line.rstrip('\n') + controllerDict["brokerid"] = brokerid + controllerDict["entity_id"] = system_test_utils.get_data_by_lookup_keyval( + tcConfigsList, "brokerid", brokerid, "entity_id") + return controllerDict + + Index: system_test/utils/replication_utils.py =================================================================== --- system_test/utils/replication_utils.py (revision 1401726) +++ system_test/utils/replication_utils.py (working copy) @@ -58,3 +58,10 @@ self.leaderAttributesDict["LEADER_ELECTION_COMPLETED_MSG"] + \ " for topic (.*?) partition (.*?) \(.*" + # Controller attributes + self.isControllerLogPattern = "Controller startup complete" + self.controllerAttributesDict = {} + self.controllerAttributesDict["CONTROLLER_STARTUP_COMPLETE_MSG"] = self.isControllerLogPattern + self.controllerAttributesDict["REGX_CONTROLLER_STARTUP_PATTERN"] = "\[(.*?)\] .* \[Controller (.*?)\]: " + \ + self.controllerAttributesDict["CONTROLLER_STARTUP_COMPLETE_MSG"] + Index: system_test/utils/system_test_utils.py =================================================================== --- system_test/utils/system_test_utils.py (revision 1401726) +++ system_test/utils/system_test_utils.py (working copy) @@ -219,7 +219,6 @@ break return pidStack - def sigterm_remote_process(hostname, pidStack): while ( len(pidStack) > 0 ): @@ -233,7 +232,6 @@ print "WARN - pid:",pid,"not found" raise - def sigkill_remote_process(hostname, pidStack): while ( len(pidStack) > 0 ): @@ -247,7 +245,36 @@ print "WARN - pid:",pid,"not found" raise +def simulate_garbage_collection_pause_in_remote_process(hostname, pidStack, pauseTimeInSeconds): + pausedPidStack = [] + # pause the processes + while len(pidStack) > 0: + pid = pidStack.pop() + pausedPidStack.append(pid) + cmdStr = "ssh " + hostname + " 'kill -SIGSTOP " + pid + "'" + + try: + logger.debug("executing command [" + cmdStr + "]", extra=d) + sys_call_return_subproc(cmdStr) + except: + print "WARN - pid:",pid,"not found" + raise + + time.sleep(int(pauseTimeInSeconds)) + + # resume execution of the processes + while len(pausedPidStack) > 0: + pid = pausedPidStack.pop() + cmdStr = "ssh " + hostname + " 'kill -SIGCONT " + pid + "'" + + try: + logger.debug("executing command [" + cmdStr + "]", extra=d) + sys_call_return_subproc(cmdStr) + except: + print "WARN - pid:",pid,"not found" + raise + def terminate_process(pidStack): while ( len(pidStack) > 0 ): pid = pidStack.pop() @@ -359,6 +386,8 @@ if hostname == "localhost" and kafkaHome == "default": clusterEntityConfigDictList[listIndex]["kafka_home"] = localKafkaHome + if hostname == "localhost" and kafkaHome == "system_test/migration_tool_testsuite/0.7": + clusterEntityConfigDictList[listIndex]["kafka_home"] = localKafkaHome + "/system_test/migration_tool_testsuite/0.7" kafkaHome = clusterEntityConfigDict["kafka_home"] javaHome = clusterEntityConfigDict["java_home"]