From d5d6031f5b9953616fa3ebd5e6bbb3a7bfe622b0 Mon Sep 17 00:00:00 2001 From: Jay Kreps Date: Thu, 11 Jul 2013 08:06:43 -0700 Subject: [PATCH] Misc. minor house-keeping fixes: add reasonable GC settings, fix up README, fix up example configs. --- README.md | 55 ++++++++++++++++++----------------------- bin/kafka-run-class.sh | 16 ++++++++++-- bin/kafka-server-start.sh | 2 +- bin/zookeeper-server-start.sh | 3 ++- config/producer.properties | 2 +- config/server.properties | 18 +++++--------- 6 files changed, 48 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index cf0d59e..9879427 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,52 @@ -# Kafka is a distributed publish/subscribe messaging system # +# Apache Kafka # -It is designed to support the following +See our [web site](http://kafka.apache.org) details on the project. -* Persistent messaging with O(1) disk structures that provide constant time performance even with many TB of stored messages. -* High-throughput: even with very modest hardware Kafka can support hundreds of thousands of messages per second. -* Explicit support for partitioning messages over Kafka servers and distributing consumption over a cluster of consumer machines while maintaining per-partition ordering semantics. -* Support for parallel data load into Hadoop. - -Kafka is aimed at providing a publish-subscribe solution that can handle all activity stream data and processing on a consumer-scale web site. This kind of activity (page views, searches, and other user actions) are a key ingredient in many of the social feature on the modern web. This data is typically handled by "logging" and ad hoc log aggregation solutions due to the throughput requirements. This kind of ad hoc solution is a viable solution to providing logging data to an offline analysis system like Hadoop, but is very limiting for building real-time processing. Kafka aims to unify offline and online processing by providing a mechanism for parallel load into Hadoop as well as the ability to partition real-time consumption over a cluster of machines. - -See our [web site](http://kafka.apache.org/) for more details on the project. - -## Contribution ## - -Kafka is a new project, and we are interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://kafka.apache.org/contact.html). - -The Kafka code is available from: - * git clone http://git-wip-us.apache.org/repos/asf/kafka.git kafka - -To contribute you can follow: - * https://cwiki.apache.org/confluence/display/KAFKA/Git+Workflow +## Building It ## To build for all supported versions of Scala: - 1. ./sbt +package To build for a particular version of Scala (either 2.8.0, 2.8.2, 2.9.1 or 2.9.2): 1. ./sbt "++2.8.0 package" *or* ./sbt "++2.8.2 package" *or* ./sbt "++2.9.1 package" *or* ./sbt "++2.9.2 package" +## Running It ## + +To run follow the instructions here: +1. http://kafka.apache.org/08/quickstart.html + +## Other Build Tips ## + Here are some useful sbt commands, to be executed at the sbt command prompt (./sbt). Prefixing with "++ " runs the command for a specific Scala version, prefixing with "+" will perform the action for all versions of Scala, and no prefix runs the command for the default (2.8.0) version of Scala. - tasks : Lists all the sbt commands and their descriptions - clean : Deletes all generated files (the target directory). - compile : Compile all the sub projects, but not create the jars - test : Run all unit tests in all sub projects - release-zip : Create all the jars, run unit tests and create a deployable release zip - package: Creates jars for src, test, docs etc - projects : List all the sub projects - project sub_project_name : Switch to a particular sub-project. For example, to switch to the core kafka code, use "project core-kafka" -Following commands can be run only on a particular sub project - - +The following commands can be run only on a particular sub project - test-only package.test.TestName : Runs only the specified test in the current sub project - run : Provides options to run any of the classes that have a main method. For example, you can switch to project java-examples, and run the examples there by executing "project java-examples" followed by "run" For more details please see the [SBT documentation](https://github.com/harrah/xsbt/wiki) +## Contribution ## + +Kafka is a new project, and we are interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://kafka.apache.org/contact.html). + +To contribute follow the instructions here: + * http://kafka.apache.org/contributing.html + +We also welcome patches for the website and documentation which can be found here: + * https://svn.apache.org/repos/asf/kafka/site + + + + diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index 1b66655..01d990c 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -25,7 +25,7 @@ base_dir=$(dirname $0)/.. SCALA_VERSION=2.8.0 # assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency" -for file in $base_dir/core/target/scala-2.8.0/*.jar; +for file in $base_dir/core/target/scala-${SCALA_VERSION}/*.jar; do CLASSPATH=$CLASSPATH:$file done @@ -64,7 +64,19 @@ else JAVA="$JAVA_HOME/bin/java" fi -$JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS -cp $CLASSPATH "$@" +GC_FILE_SUFFIX='-gc.log' +GC_LOG_FILE_NAME='' +JVM_PERFORMANCE_OPTS="-XX:+UseCompressedOops -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC" + +if [ "$1" = "daemon" ] ; then + shift + GC_LOG_FILE_NAME=$1$GC_FILE_SUFFIX + shift + JVM_PERFORMANCE_OPTS="$JVM_PERFORMANCE_OPTS -Xloggc:$base_dir/$GC_LOG_FILE_NAME -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps" +fi + +echo $CLASSPATH +$JAVA $KAFKA_OPTS $KAFKA_JMX_OPTS $JVM_PERFORMANCE_OPTS -cp $CLASSPATH "$@" exitval=$? diff --git a/bin/kafka-server-start.sh b/bin/kafka-server-start.sh index 307f76c..0b25185 100755 --- a/bin/kafka-server-start.sh +++ b/bin/kafka-server-start.sh @@ -22,4 +22,4 @@ fi export JMX_PORT=${JMX_PORT:-9999} -$(dirname $0)/kafka-run-class.sh kafka.Kafka $@ +$(dirname $0)/kafka-run-class.sh daemon kafkaServer kafka.Kafka $@ diff --git a/bin/zookeeper-server-start.sh b/bin/zookeeper-server-start.sh index 184a10b..f53b7dc 100755 --- a/bin/zookeeper-server-start.sh +++ b/bin/zookeeper-server-start.sh @@ -20,4 +20,5 @@ then exit 1 fi -$(dirname $0)/kafka-run-class.sh org.apache.zookeeper.server.quorum.QuorumPeerMain $@ +$(dirname $0)/kafka-run-class.sh daemon zookeeper org.apache.zookeeper.server.quorum.QuorumPeerMain $@ + diff --git a/config/producer.properties b/config/producer.properties index 162b8a6..52a7611 100644 --- a/config/producer.properties +++ b/config/producer.properties @@ -16,7 +16,7 @@ ############################# Producer Basics ############################# -# list of brokers used for bootstrapping +# list of brokers used for bootstrapping knowledge about the rest of the cluster # format: host1:port1,host2:port2 ... metadata.broker.list=localhost:9092 diff --git a/config/server.properties b/config/server.properties index 0589c71..977b8da 100644 --- a/config/server.properties +++ b/config/server.properties @@ -47,8 +47,8 @@ socket.request.max.bytes=104857600 ############################# Log Basics ############################# -# The directory under which to store log files -log.dir=/tmp/kafka-logs +# A comma seperated list of directories under which to store log files +log.dirs=/tmp/kafka-logs # The number of logical partitions per topic per server. More partitions allow greater parallelism # for consumption, but also mean more files. @@ -56,12 +56,12 @@ num.partitions=2 ############################# Log Flush Policy ############################# -# The following configurations control the flush of data to disk. This is the most +# The following configurations control the flush of data to disk. This is among the most # important performance knob in kafka. # There are a few important trade-offs here: -# 1. Durability: Unflushed data is at greater risk of loss in the event of a crash. -# 2. Latency: Data is not made available to consumers until it is flushed (which adds latency). -# 3. Throughput: The flush is generally the most expensive operation. +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. # The settings below allow one to configure the flush policy to flush data after a period of time or # every N messages (or both). This can be done globally and overridden on a per-topic basis. @@ -107,10 +107,4 @@ zookeeper.connect=localhost:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=1000000 -# metrics reporter properties -kafka.metrics.polling.interval.secs=5 -kafka.metrics.reporters=kafka.metrics.KafkaCSVMetricsReporter -kafka.csv.metrics.dir=/tmp/kafka_metrics -# Disable csv reporting by default. -kafka.csv.metrics.reporter.enabled=false -- 1.7.10.2 (Apple Git-33)