From a3c8e3f63082fdb1fed1ec1c781acb79f22a3994 Mon Sep 17 00:00:00 2001 From: Stevo Slavic Date: Wed, 9 Apr 2014 08:54:00 +0200 Subject: [PATCH 1/3] Add windows bat file kafka-topics.bat Use %~dp0 instead of %cd% as %~dp0 always refers to the folder the script exists in. Switching to %~dp0 means the user can execute the bat files from any directory. --- bin/windows/kafka-console-consumer.bat | 4 +- bin/windows/kafka-console-producer.bat | 5 +- bin/windows/kafka-run-class.bat | 86 ++++++++++++++++++---------------- bin/windows/kafka-server-start.bat | 15 +++--- bin/windows/kafka-topics.bat | 17 +++++++ bin/windows/zookeeper-server-start.bat | 10 ++-- 6 files changed, 83 insertions(+), 54 deletions(-) create mode 100644 bin/windows/kafka-topics.bat diff --git a/bin/windows/kafka-console-consumer.bat b/bin/windows/kafka-console-consumer.bat index a287b9e..94b20a4 100644 --- a/bin/windows/kafka-console-consumer.bat +++ b/bin/windows/kafka-console-consumer.bat @@ -15,6 +15,6 @@ rem See the License for the specific language governing permissions and rem limitations under the License. SetLocal -set KAFKA_OPTS=-Xmx512M -server -Dcom.sun.management.jmxremote -Dlog4j.configuration=file:"%CD%\kafka-console-consumer-log4j.properties" -kafka-run-class.bat kafka.consumer.ConsoleConsumer %* +set KAFKA_HEAP_OPTS=-Xmx512M +%~dp0kafka-run-class.bat kafka.consumer.ConsoleConsumer %* EndLocal diff --git a/bin/windows/kafka-console-producer.bat b/bin/windows/kafka-console-producer.bat index b321ee2..b116e64 100644 --- a/bin/windows/kafka-console-producer.bat +++ b/bin/windows/kafka-console-producer.bat @@ -14,4 +14,7 @@ rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. rem See the License for the specific language governing permissions and rem limitations under the License. -kafka-run-class.bat kafka.producer.ConsoleProducer %* +SetLocal +set KAFKA_HEAP_OPTS=-Xmx512M +%~dp0kafka-run-class.bat kafka.producer.ConsoleProducer %* +EndLocal diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat index f274cd7..4a3597e 100644 --- a/bin/windows/kafka-run-class.bat +++ b/bin/windows/kafka-run-class.bat @@ -17,76 +17,82 @@ rem limitations under the License. setlocal enabledelayedexpansion IF [%1] EQU [] ( - echo "USAGE: $0 classname [opts]" - goto :eof + echo USAGE: %0 classname [opts] + EXIT /B 1 ) -set BASE_DIR=%CD%\.. +rem Using pushd popd to set BASE_DIR to the absolute path +pushd %~dp0..\.. +set BASE_DIR=%CD% +popd set CLASSPATH= -echo %BASE_DIR% -set ivyPath=%USERPROFILE%\.ivy2\cache - -set snappy=%ivyPath%/org.xerial.snappy/snappy-java/bundles/snappy-java-1.0.5.jar - call :concat %snappy% - -set library=%ivyPath%/org.scala-lang/scala-library/jars/scala-library-2.8.0.jar - call :concat %library% - -set compiler=%ivyPath%/org.scala-lang/scala-compiler/jars/scala-compiler-2.8.0.jar - call :concat %compiler% - -set log4j=%ivyPath%/log4j/log4j/jars/log4j-1.2.15.jar - call :concat %log4j% - -set slf=%ivyPath%/org.slf4j/slf4j-api/jars/slf4j-api-1.6.4.jar - call :concat %slf% - -set zookeeper=%ivyPath%/org.apache.zookeeper/zookeeper/jars/zookeeper-3.3.4.jar - call :concat %zookeeper% - -set jopt=%ivyPath%/net.sf.jopt-simple/jopt-simple/jars/jopt-simple-3.2.jar - call :concat %jopt% +IF ["%SCALA_VERSION%"] EQU [""] ( + set SCALA_VERSION=2.8.0 +) -for %%i in (%BASE_DIR%\core\target\scala-2.8.0\*.jar) do ( +rem Assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency" +for %%i in (%BASE_DIR%\core\target\scala-%SCALA_VERSION%\*.jar) do ( call :concat %%i ) -for %%i in (%BASE_DIR%\core\lib\*.jar) do ( +rem Classpath addition for release +for %%i in (%BASE_DIR%\libs\*.jar) do ( call :concat %%i ) -for %%i in (%BASE_DIR%\perf\target\scala-2.8.0/kafka*.jar) do ( +for %%i in (%BASE_DIR%\perf\target\scala-%SCALA_VERSION%\kafka*.jar) do ( call :concat %%i ) -IF ["%KAFKA_JMX_OPTS%"] EQU [""] ( - set KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false +for %%i in (%BASE_DIR%\kafka_*.jar) do ( + call :concat %%i ) -IF ["%KAFKA_OPTS%"] EQU [""] ( - set KAFKA_OPTS=-Xmx512M -server -Dlog4j.configuration=file:"%BASE_DIR%\config\log4j.properties" +rem JMX settings +IF ["%KAFKA_JMX_OPTS%"] EQU [""] ( + set KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false ) +rem JMX port to use IF ["%JMX_PORT%"] NEQ [""] ( set KAFKA_JMX_OPTS=%KAFKA_JMX_OPTS% -Dcom.sun.management.jmxremote.port=%JMX_PORT% ) +rem Log4j settings +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%\config\tools-log4j.properties +) + +rem Generic jvm settings you want to add +IF ["%KAFKA_OPTS%"] EQU [""] ( + set KAFKA_OPTS= +) + +rem Which java to use IF ["%JAVA_HOME%"] EQU [""] ( set JAVA=java ) ELSE ( set JAVA="%JAVA_HOME%/bin/java" ) -set SEARCHTEXT=\bin\.. -set REPLACETEXT= -set CLASSPATH=!CLASSPATH:%SEARCHTEXT%=%REPLACETEXT%! -set COMMAND= %JAVA% %KAFKA_OPTS% %KAFKA_JMX_OPTS% -cp %CLASSPATH% %* -set SEARCHTEXT=-cp ; -set REPLACETEXT=-cp -set COMMAND=!COMMAND:%SEARCHTEXT%=%REPLACETEXT%! +rem Memory options +IF ["%KAFKA_HEAP_OPTS%"] EQU [""] ( + set KAFKA_HEAP_OPTS=-Xmx256M +) + +rem JVM performance options +IF ["%KAFKA_JVM_PERFORMANCE_OPTS%"] EQU [""] ( + set KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseCompressedOops -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC -Djava.awt.headless=true +) + +set COMMAND=%JAVA% %KAFKA_HEAP_OPTS% %KAFKA_JVM_PERFORMANCE_OPTS% %KAFKA_JMX_OPTS% %KAFKA_LOG4J_OPTS% -cp %CLASSPATH% %KAFKA_OPTS% %* +rem echo. +rem echo %COMMAND% +rem echo. %COMMAND% +goto :eof :concat set CLASSPATH=%CLASSPATH%;"%1" \ No newline at end of file diff --git a/bin/windows/kafka-server-start.bat b/bin/windows/kafka-server-start.bat index 7db76dd..484e136 100644 --- a/bin/windows/kafka-server-start.bat +++ b/bin/windows/kafka-server-start.bat @@ -15,13 +15,12 @@ rem See the License for the specific language governing permissions and rem limitations under the License. IF [%1] EQU [] ( - echo "USAGE: $0 server.properties [consumer.properties producer.properties]" - goto :eof + echo USAGE: %0 server.properties + EXIT /B 1 ) -IF [%JMX_PORT%] EQU [] ( - echo Set JMX_PORT to default value : 9999 - set JMX_PORT=9999 -) - -kafka-run-class.bat kafka.Kafka %* \ No newline at end of file +SetLocal +set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:~dp0../../config/log4j.properties +set KAFKA_HEAP_OPTS=-Xmx1G -Xms1G +%~dp0kafka-run-class.bat kafka.Kafka %* +EndLocal diff --git a/bin/windows/kafka-topics.bat b/bin/windows/kafka-topics.bat new file mode 100644 index 0000000..f1a9e64 --- /dev/null +++ b/bin/windows/kafka-topics.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +%~dp0kafka-run-class.bat kafka.admin.TopicCommand %* diff --git a/bin/windows/zookeeper-server-start.bat b/bin/windows/zookeeper-server-start.bat index 0dd40ba..9836283 100644 --- a/bin/windows/zookeeper-server-start.bat +++ b/bin/windows/zookeeper-server-start.bat @@ -15,8 +15,12 @@ rem See the License for the specific language governing permissions and rem limitations under the License. IF [%1] EQU [] ( - echo "USAGE: $0 zookeeper.properties" - goto :eof + echo USAGE: %0 zookeeper.properties + EXIT /B 1 ) -kafka-run-class.bat org.apache.zookeeper.server.quorum.QuorumPeerMain %* +SetLocal +set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties +set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M +%~dp0kafka-run-class.bat org.apache.zookeeper.server.quorum.QuorumPeerMain %* +EndLocal -- 1.8.4.msysgit.0 From db3403d83c8cc913d5f32d83a969ef0a2e937d5d Mon Sep 17 00:00:00 2001 From: Stevo Slavic Date: Wed, 9 Apr 2014 10:01:36 +0200 Subject: [PATCH 2/3] Changed classpath element references as consequence of migration from SBT to Gradle --- bin/windows/kafka-run-class.bat | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat index 4a3597e..1a3e468 100644 --- a/bin/windows/kafka-run-class.bat +++ b/bin/windows/kafka-run-class.bat @@ -25,27 +25,49 @@ rem Using pushd popd to set BASE_DIR to the absolute path pushd %~dp0..\.. set BASE_DIR=%CD% popd -set CLASSPATH= +set CLASSPATH=. IF ["%SCALA_VERSION%"] EQU [""] ( set SCALA_VERSION=2.8.0 ) -rem Assume all dependencies have been packaged into one jar with sbt-assembly's task "assembly-package-dependency" -for %%i in (%BASE_DIR%\core\target\scala-%SCALA_VERSION%\*.jar) do ( +rem Classpath addition for kafka-core dependencies +for %%i in (%BASE_DIR%\core\build\dependant-libs-%SCALA_VERSION%\*.jar) do ( call :concat %%i ) -rem Classpath addition for release -for %%i in (%BASE_DIR%\libs\*.jar) do ( +rem Classpath addition for kafka-perf dependencies +for %%i in (%BASE_DIR%\perf\build\dependant-libs-%SCALA_VERSION%\*.jar) do ( + call :concat %%i +) + +rem Classpath addition for kafka-clients +for %%i in (%BASE_DIR%\clients\build\libs\kafka-clients-*.jar) do ( + call :concat %%i +) + +rem Classpath addition for kafka-examples +for %%i in (%BASE_DIR%\examples\build\libs\kafka-examples-*.jar) do ( call :concat %%i ) -for %%i in (%BASE_DIR%\perf\target\scala-%SCALA_VERSION%\kafka*.jar) do ( +rem Classpath addition for contrib/hadoop-consumer +for %%i in (%BASE_DIR%\contrib\hadoop-consumer\build\libs\kafka-hadoop-consumer-*.jar) do ( + call :concat %%i +) + +rem Classpath addition for contrib/hadoop-producer +for %%i in (%BASE_DIR%\contrib\hadoop-producer\build\libs\kafka-hadoop-producer-*.jar) do ( + call :concat %%i +) + +rem Classpath addition for release +for %%i in (%BASE_DIR%\libs\*.jar) do ( call :concat %%i ) -for %%i in (%BASE_DIR%\kafka_*.jar) do ( +rem Classpath addition for core +for %%i in (%BASE_DIR%\core\builds\libs\kafka_%SCALA_VERSION%*.jar) do ( call :concat %%i ) -- 1.8.4.msysgit.0 From 427a9fd90da89546585d606db80a261bd7de181e Mon Sep 17 00:00:00 2001 From: Stevo Slavic Date: Wed, 9 Apr 2014 10:28:32 +0200 Subject: [PATCH 3/3] Implemented error handling in windows/kafka-run-class.bat, when empty classpath is detected --- bin/windows/kafka-run-class.bat | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/windows/kafka-run-class.bat b/bin/windows/kafka-run-class.bat index 1a3e468..be1c471 100644 --- a/bin/windows/kafka-run-class.bat +++ b/bin/windows/kafka-run-class.bat @@ -25,7 +25,7 @@ rem Using pushd popd to set BASE_DIR to the absolute path pushd %~dp0..\.. set BASE_DIR=%CD% popd -set CLASSPATH=. +set CLASSPATH= IF ["%SCALA_VERSION%"] EQU [""] ( set SCALA_VERSION=2.8.0 @@ -108,6 +108,11 @@ IF ["%KAFKA_JVM_PERFORMANCE_OPTS%"] EQU [""] ( set KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseCompressedOops -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:+CMSScavengeBeforeRemark -XX:+DisableExplicitGC -Djava.awt.headless=true ) +IF [%CLASSPATH%] EQU [] ( + echo Classpath is empty. Please build the project first e.g. by running 'gradlew jarAll' + EXIT /B 2 +) + set COMMAND=%JAVA% %KAFKA_HEAP_OPTS% %KAFKA_JVM_PERFORMANCE_OPTS% %KAFKA_JMX_OPTS% %KAFKA_LOG4J_OPTS% -cp %CLASSPATH% %KAFKA_OPTS% %* rem echo. rem echo %COMMAND% -- 1.8.4.msysgit.0