Index: perf/config/log4j.properties
===================================================================
--- perf/config/log4j.properties	(revision 0)
+++ perf/config/log4j.properties	(revision 0)
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, fileAppender
+
+log4j.appender.fileAppender=org.apache.log4j.FileAppender
+log4j.appender.fileAppender.File=perf.log
+log4j.appender.fileAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.fileAppender.layout.ConversionPattern=%m %n 
+
+# Turn on all our debugging info
+log4j.logger.kafka.perf.ProducerPerformance=INFO
+log4j.logger.kafka.producer=INFO
+#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG
+
Index: perf/bin/consumer-throughput-over-fetch-size.sh
===================================================================
--- perf/bin/consumer-throughput-over-fetch-size.sh	(revision 0)
+++ perf/bin/consumer-throughput-over-fetch-size.sh	(revision 0)
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+if [ $# -lt 1 ];
+then
+        echo "USAGE: $0 topic [zkconnect]" 
+        exit 1
+fi
+
+# default zookeeper is at localhost:2181
+zkconnect=127.0.0.1
+port=2181
+
+if [ $# -eq 2 ];
+then
+  zkconnect=$2	
+fi
+
+perf_dir=$(dirname $0)/..
+topic=$1
+perf_log=$perf_dir/consumer-throughput-over-fetch-size.csv
+
+# delete the perf.log file
+rm $perf_log
+touch $perf_log
+
+echo "start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec" >> $perf_log
+
+# fetch size = 100K 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --fetch-size 102400 --avg >> $perf_log 
+ 
+# fetch size = 200K 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --fetch-size 204800 --avg >> $perf_log
+ 
+# fetch size = 500K 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --fetch-size 512000 --avg >> $perf_log
+
+# fetch size = 1MB 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --avg >> $perf_log
+
+# fetch size = 2MB 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --fetch-size 2097152 --avg >> $perf_log
+
+# fetch size = 5MB 
+$perf_dir/bin/kafka-consumer-perf-test.sh --topic $topic --zookeeper $zkconnect --fetch-size 5242880 --avg >> $perf_log
+

Property changes on: perf/bin/consumer-throughput-over-fetch-size.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/producer-throughput-over-batch-size.sh
===================================================================
--- perf/bin/producer-throughput-over-batch-size.sh	(revision 0)
+++ perf/bin/producer-throughput-over-batch-size.sh	(revision 0)
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+
+# default brokerid, server and port is 0:localhost:9092
+brokerid=0
+server=localhost
+port=9093
+
+if [ $# -gt 3 ];
+then
+ echo "USAGE: $0 [brokerid] [server] [port]"
+ exit 1
+fi
+
+if [ $# -eq 3 ];
+then
+  brokerid=$1
+  server=$2
+  port=$3
+fi
+
+echo "Producing to broker id $brokerid on $server:$port"
+
+perf_dir=$(dirname $0)/..
+perf_log=$perf_dir/producer-throughput-over-batch-size.csv
+
+# delete the perf.csv file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec" >> $perf_log
+
+# no compression
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+# batch size = 1
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 50
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 50 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 200
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 200 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 500
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 500 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 1000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 5000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 5000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 10000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 10000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# gzip compression
+# batch size = 1
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 50
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 50 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 200
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 200 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 500
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 500 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 1000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 5000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 5000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 10000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 10000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+

Property changes on: perf/bin/producer-throughput-over-batch-size.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/all-zk-producer-tests.sh
===================================================================
--- perf/bin/all-zk-producer-tests.sh	(revision 0)
+++ perf/bin/all-zk-producer-tests.sh	(revision 0)
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+base_dir=$(dirname $0)
+$base_dir/zk-producer-throughput-over-batch-size.sh $@ 
+$base_dir/zk-producer-throughput-over-data-accumulated.sh $@

Property changes on: perf/bin/all-zk-producer-tests.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/zk-producer-throughput-over-batch-size.sh
===================================================================
--- perf/bin/zk-producer-throughput-over-batch-size.sh	(revision 0)
+++ perf/bin/zk-producer-throughput-over-batch-size.sh	(revision 0)
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+
+# default server and port is localhost:9092
+zkconnect=127.0.0.1:2181
+
+if [ $# -gt 1 ];
+then
+ echo "USAGE: $0 [zkconnect]"
+ exit 1
+fi
+
+if [ $# -eq 1 ];
+then
+  zkconnect=$1
+fi
+
+echo "Producing to kafka clusters registered at $zkconnect"
+
+perf_dir=$(dirname $0)/..
+perf_log=$perf_dir/zk-producer-throughput-over-batch-size.csv
+
+# delete the perf.csv file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "start.time, end.time, compression, message.size, batch.size, total.data.sent.in.MB, MB.sec, total.data.sent.in.nMsg, nMsg.sec" >> $perf_log
+
+# no compression
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+# batch size = 1
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 50
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 50 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 200
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 200 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 500
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 500 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 1000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 5000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 5000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+
+# batch size = 10000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 10000 --messages 5000000 --threads 1 --async --avg >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# gzip compression
+# batch size = 1
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo zk.connect=$zkconnect --batch-size 1 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 50
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 50 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 200
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 200 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 500
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 500 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 1000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 5000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 5000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+
+# batch size = 10000
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 10000 --messages 5000000 --threads 1 --async --avg --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+

Property changes on: perf/bin/zk-producer-throughput-over-batch-size.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/kafka-consumer-perf-test.sh
===================================================================
--- perf/bin/kafka-consumer-perf-test.sh	(revision 0)
+++ perf/bin/kafka-consumer-perf-test.sh	(revision 0)
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$(dirname $0)/kafka-run-class.sh kafka.perf.ConsumerPerformance $@

Property changes on: perf/bin/kafka-consumer-perf-test.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/producer-throughput-over-data-accumulated.sh
===================================================================
--- perf/bin/producer-throughput-over-data-accumulated.sh	(revision 0)
+++ perf/bin/producer-throughput-over-data-accumulated.sh	(revision 0)
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# default server and port is localhost:9092
+brokerid=0
+server=localhost
+port=9093
+
+if [ $# -gt 3 ];
+then
+ echo "USAGE: $0 [brokerid] [server] [port]"
+ exit 1
+fi
+
+if [ $# -eq 3 ];
+then
+  brokerid=$1
+  server=$2
+  port=$3
+fi
+
+echo "Producing to broker id $brokerid on $server:$port"
+
+perf_dir=$(dirname $0)/..
+perf_log=$perf_dir/producer-throughput-over-data-accumulated.csv
+
+# delete the perf.csv file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "time, compression, thread.id, message.size, batch.size, data.sent.in.MB, MB.sec, data.sent.in.nMsg, nMsg.sec" >> $perf_log
+
+# no compression
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 50
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 50 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 200
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 200 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 500
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 500 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 1000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 5000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 5000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 10000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 10000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# gzip compression
+# batch size 1
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 50
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 50 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 200
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 200 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 500
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 500 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 1000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 1000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 5000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 5000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 10000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo broker.list=$brokerid:$server:$port --batch-size 10000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log

Property changes on: perf/bin/producer-throughput-over-data-accumulated.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/producer-throughput-over-num-topics.sh
===================================================================
--- perf/bin/producer-throughput-over-num-topics.sh	(revision 0)
+++ perf/bin/producer-throughput-over-num-topics.sh	(revision 0)
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# This script reads several producer-throughput-over-data-accumulated.csv files, 
+# each in a directory named after the number of topics on the server
+# For example, in a top level directory, say producer-throughput-vs-num-topics, there
+# should be directories named 1, 50, 100, 500, 1000, 5000. Each of them will have a
+# producer-throughput-over-data-accumulated.csv file
+
+if [ $# -lt 1 ];
+then
+ echo "USAGE: $0 input-directory"
+ exit 1
+fi
+
+perf_dir=$1
+perf_dir=`cd "$perf_dir"; pwd`
+perf_log=$perf_dir/producer-throughput-over-num-topics.csv
+
+# delete the perf.csv file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "compression, thread.id, message.size, batch.size, data.sent.in.MB, MB.sec, data.sent.in.nMsg, nMsg.sec, num.topics" >> $perf_log
+
+for d in $(ls $perf_dir)
+do
+  if [ -d $d ];
+  then
+    # get the directory name that specifies the number of topics for the run
+    echo "Num topics directory is $d"
+    # this directory should have a file called producer-throughput-over-data-accumulated.csv
+    for f in $(find $d -name "*csv")
+    do
+      perf_file_name=${f##*/}
+      if [ $perf_file_name == "producer-throughput-over-data-accumulated.csv" ];
+      then
+        echo "Producer perf file: $f"
+        cat $perf_dir/$f | egrep -v "Run|MB" | sed "s/.*/&, $d/" >> $perf_log
+      fi
+    done
+  fi
+done
+

Property changes on: perf/bin/producer-throughput-over-num-topics.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/simple-consumer-throughput-over-fetch-size.sh
===================================================================
--- perf/bin/simple-consumer-throughput-over-fetch-size.sh	(revision 0)
+++ perf/bin/simple-consumer-throughput-over-fetch-size.sh	(revision 0)
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ $# -lt 1 ];
+then
+        echo "USAGE: $0 topic [server] [port]" 
+        exit 1
+fi
+
+# default server and port is localhost:9093
+server=localhost
+port=9093
+
+if [ $# -eq 3 ];
+then
+  server=$2
+  port=$3	
+fi
+
+perf_dir=$(dirname $0)/..
+topic=$1
+perf_log=$perf_dir/simple-consumer-throughput-over-fetch-size.csv
+
+# delete the perf.log file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "start.time, end.time, fetch.size, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec" >> $perf_log
+
+# fetch size = 100K 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --fetch-size 102400 --avg >> $perf_log 
+ 
+# fetch size = 200K 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --fetch-size 204800 --avg >> $perf_log
+ 
+# fetch size = 500K 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --fetch-size 512000 --avg >> $perf_log
+
+# fetch size = 1MB 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --avg >> $perf_log
+
+# fetch size = 2MB 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --fetch-size 2097152 --avg >> $perf_log
+
+# fetch size = 5MB 
+$perf_dir/bin/kafka-simple-consumer-perf-test.sh --topic $topic --server kafka://$server:$port --fetch-size 5242880 --avg >> $perf_log
+

Property changes on: perf/bin/simple-consumer-throughput-over-fetch-size.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/zk-producer-throughput-over-data-accumulated.sh
===================================================================
--- perf/bin/zk-producer-throughput-over-data-accumulated.sh	(revision 0)
+++ perf/bin/zk-producer-throughput-over-data-accumulated.sh	(revision 0)
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# default server and port is localhost:9092
+zkconnect=127.0.0.1:2181
+
+if [ $# -gt 1 ];
+then
+ echo "USAGE: $0 [zkconnect]"
+ exit 1
+fi
+
+if [ $# -eq 1 ];
+then
+  zkconnect=$1
+fi
+
+echo "Producing to kafka clusters registered at $zkconnect"
+perf_dir=$(dirname $0)/..
+perf_log=$perf_dir/zk-producer-throughput-over-data-accumulated.csv
+
+# delete the perf.csv file
+rm $perf_log
+touch $perf_log
+
+# add the perf header
+echo "time, compression, thread.id, message.size, batch.size, data.sent.in.MB, MB.sec, data.sent.in.nMsg, nMsg.sec" >> $perf_log
+
+# no compression
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 50
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 50 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 200
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 200 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 500
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 500 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 1000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 5000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 5000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 10000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 10000 --messages 5000000 --message-size 200 --threads 1 --async  >> $perf_log 
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# gzip compression
+# batch size 1
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo zk.connect=$zkconnect --batch-size 1 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 50
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 50 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 200
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 200 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 500
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerfPerf --brokerinfo zk.connect=$zkconnect --batch-size 500 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 1000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 1000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 5000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 5000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log
+
+# batch size 10000
+echo "Run start time: " `date +%H:%M:%S` >> $perf_log
+$perf_dir/bin/kafka-producer-perf-test.sh --topic testAsyncCompressedProducerPerf --brokerinfo zk.connect=$zkconnect --batch-size 10000 --messages 5000000 --message-size 200 --threads 1 --async  --compression-codec 1 >> $perf_log
+echo "Run end time: " `date +%H:%M:%S` >> $perf_log

Property changes on: perf/bin/zk-producer-throughput-over-data-accumulated.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/kafka-run-class.sh
===================================================================
--- perf/bin/kafka-run-class.sh	(revision 0)
+++ perf/bin/kafka-run-class.sh	(revision 0)
@@ -0,0 +1,66 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+  echo "USAGE: $0 classname [opts]"
+  exit 1
+fi
+
+base_dir=$(dirname $0)/../..
+
+for file in $base_dir/project/boot/scala-2.8.0/lib/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/core/target/scala_2.8.0/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/perf/target/scala_2.8.0/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/core/lib/*.jar;
+do
+  CLASSPATH=$CLASSPATH:$file
+done
+
+for file in $base_dir/core/lib_managed/scala_2.8.0/compile/*.jar;
+do
+  if [ ${file##*/} != "sbt-launch.jar" ]; then
+    CLASSPATH=$CLASSPATH:$file
+  fi
+done
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+if [ -z "$KAFKA_OPTS" ]; then
+  KAFKA_OPTS="-Xmx5G -server  -Dlog4j.configuration=file:$base_dir/perf/config/log4j.properties"
+fi
+if [  $JMX_PORT ]; then
+  KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
+fi
+if [ -z "$JAVA_HOME" ]; then
+  JAVA="java"
+else
+  JAVA="$JAVA_HOME/bin/java"
+fi
+
+$JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH $@

Property changes on: perf/bin/kafka-run-class.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/all-producer-perf-tests.sh
===================================================================
--- perf/bin/all-producer-perf-tests.sh	(revision 0)
+++ perf/bin/all-producer-perf-tests.sh	(revision 0)
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+base_dir=$(dirname $0)
+$base_dir/producer-throughput-over-batch-size.sh $@ 
+$base_dir/producer-throughput-over-data-accumulated.sh $@

Property changes on: perf/bin/all-producer-perf-tests.sh
___________________________________________________________________
Added: svn:executable
   + *

\ No newline at end of file
Index: perf/bin/kafka-simple-consumer-perf-test.sh
===================================================================
--- perf/bin/kafka-simple-consumer-perf-test.sh	(revision 0)
+++ perf/bin/kafka-simple-consumer-perf-test.sh	(revision 0)
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$(dirname $0)/kafka-run-class.sh kafka.perf.SimpleConsumerPerformance $@

Property changes on: perf/bin/kafka-simple-consumer-perf-test.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: perf/bin/kafka-producer-perf-test.sh
===================================================================
--- perf/bin/kafka-producer-perf-test.sh	(revision 0)
+++ perf/bin/kafka-producer-perf-test.sh	(revision 0)
@@ -0,0 +1,17 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#    http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$(dirname $0)/kafka-run-class.sh kafka.perf.ProducerPerformance $@

Property changes on: perf/bin/kafka-producer-perf-test.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: project/build/KafkaProject.scala
===================================================================
--- project/build/KafkaProject.scala	(revision 1189534)
+++ project/build/KafkaProject.scala	(working copy)
@@ -21,6 +21,7 @@
   lazy val core = project("core", "core-kafka", new CoreKafkaProject(_))
   lazy val examples = project("examples", "java-examples", new KafkaExamplesProject(_), core)
   lazy val contrib = project("contrib", "contrib", new ContribProject(_))
+  lazy val perf = project("perf", "perf", new KafkaPerfProject(_))
 
   lazy val releaseZipTask = core.packageDistTask
 
@@ -58,9 +59,6 @@
       </dependency>
     </dependencies>
 
-    override def repositories = Set(ScalaToolsSnapshots, "JBoss Maven 2 Repository" at "http://repository.jboss.com/maven2",
-      "Oracle Maven 2 Repository" at "http://download.oracle.com/maven", "maven.org" at "http://repo2.maven.org/maven2/")
-
     override def artifactID = "kafka"
     override def filterScalaJars = false
 
@@ -123,6 +121,29 @@
 
   }
 
+  class KafkaPerfProject(info: ProjectInfo) extends DefaultProject(info)
+     with IdeaProject
+     with CoreDependencies {
+    val perfPackageAction = packageAllAction
+    val dependsOnCore = core
+
+  //The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required
+  // some dependencies on various sun and javax packages.
+   override def ivyXML =
+    <dependencies>
+      <exclude module="javax"/>
+      <exclude module="jmxri"/>
+      <exclude module="jmxtools"/>
+      <exclude module="mail"/>
+      <exclude module="jms"/>
+    </dependencies>
+
+    override def artifactID = "kafka-perf"
+    override def filterScalaJars = false
+    override def javaCompileOptions = super.javaCompileOptions ++
+      List(JavaCompileOption("-Xlint:unchecked"))
+  }
+
   class KafkaExamplesProject(info: ProjectInfo) extends DefaultProject(info)
      with IdeaProject
      with CoreDependencies {