diff --git a/README.md b/README.md index e3fea22..ec80083 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,78 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Apache Kafka # +Apache Kafka +================= See our [web site](http://kafka.apache.org) for details on the project. -## Building a jar and running it ## -1. ./gradlew jar -2. Follow instuctions in http://kafka.apache.org/documentation.html#quickstart +### Building a jar and running it ### + ./gradlew jar -## Running unit tests ## -./gradlew test +Follow instuctions in http://kafka.apache.org/documentation.html#quickstart -## Forcing re-running unit tests w/o code change ## -./gradlew cleanTest test +### Running unit tests ### + ./gradlew test -## Running a particular unit test ## -./gradlew -Dtest.single=RequestResponseSerializationTest core:test +### Forcing re-running unit tests w/o code change ### + ./gradlew cleanTest test + +### Running a particular unit test ### + ./gradlew -Dtest.single=RequestResponseSerializationTest core:test + +### Building a binary release gzipped tar ball ### + ./gradlew clean + ./gradlew releaseTarGz -## Building a binary release gzipped tar ball ## -./gradlew clean -./gradlew releaseTarGz The release file can be found inside ./core/build/distributions/. -## Cleaning the build ## -./gradlew clean +### Cleaning the build ### + ./gradlew clean + +### Running a task on a particular version of Scala #### +either 2.8.0, 2.8.2, 2.9.1, 2.9.2 or 2.10.1) (If building a jar with a version other than 2.8.0, the scala version variable in bin/kafka-run-class.sh needs to be changed to run quick start.) + ./gradlew -PscalaVersion=2.9.1 jar + ./gradlew -PscalaVersion=2.9.1 test + ./gradlew -PscalaVersion=2.9.1 releaseTarGz + +### Running a task for a specific project ### +This is for 'core', 'perf', 'contrib:hadoop-consumer', 'contrib:hadoop-producer', 'examples' and 'clients' + ./gradlew core:jar + ./gradlew core:test -## Running a task on a particular version of Scala (either 2.8.0, 2.8.2, 2.9.1, 2.9.2 or 2.10.1) ## -## (If building a jar with a version other than 2.8.0, the scala version variable in bin/kafka-run-class.sh needs to be changed to run quick start.) ## -./gradlew -PscalaVersion=2.9.1 jar -./gradlew -PscalaVersion=2.9.1 test -./gradlew -PscalaVersion=2.9.1 releaseTarGz +### Listing all gradle tasks ### + ./gradlew tasks -## Running a task for a specific project in 'core', 'perf', 'contrib:hadoop-consumer', 'contrib:hadoop-producer', 'examples', 'clients' ## -./gradlew core:jar -./gradlew core:test +### Building IDE project #### + ./gradlew eclipse + ./gradlew idea -## Listing all gradle tasks ## -./gradlew tasks +### Building the jar for all scala versions and for all projects ### + ./gradlew jarAll -# Building IDE project ## -./gradlew eclipse -./gradlew idea +### Running unit tests for all scala versions and for all projects ### + ./gradlew testAll -# Building the jar for all scala versions and for all projects ## -./gradlew jarAll +### Building a binary release gzipped tar ball for all scala versions ### + ./gradlew releaseTarGzAll -## Running unit tests for all scala versions and for all projects ## -./gradlew testAll +### Publishing the jar for all version of Scala and for all projects to maven ### + ./gradlew uploadArchivesAll -## Building a binary release gzipped tar ball for all scala versions ## -./gradlew releaseTarGzAll +Please note for this to work you need to create/update `~/.gradle/gradle.properties` and assign the following variables -## Publishing the jar for all version of Scala and for all projects to maven (To test locally, change mavenUrl in gradle.properties to a local dir.) ## -./gradlew uploadArchivesAll + mavenUrl= + mavenUsername= + mavenPassword= + signing.keyId= + signing.password= + signing.secretKeyRingFile= -## Building the test jar ## -./gradlew testJar +### Building the test jar ### + ./gradlew testJar -## Determining how transitive dependencies are added ## -./gradlew core:dependencies --configuration runtime +### Determining how transitive dependencies are added ### + ./gradlew core:dependencies --configuration runtime -## Contribution ## +### Contribution ### -Kafka is a new project, and we are interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://kafka.apache.org/contact.html). +Apache Kafka interested in building the community; we would welcome any thoughts or [patches](https://issues.apache.org/jira/browse/KAFKA). You can reach us [on the Apache mailing lists](http://kafka.apache.org/contact.html). To contribute follow the instructions here: * http://kafka.apache.org/contributing.html diff --git a/core/src/main/scala/kafka/consumer/TopicCount.scala b/core/src/main/scala/kafka/consumer/TopicCount.scala index e332633..0be03c0 100644 --- a/core/src/main/scala/kafka/consumer/TopicCount.scala +++ b/core/src/main/scala/kafka/consumer/TopicCount.scala @@ -19,7 +19,7 @@ package kafka.consumer import scala.collection._ import org.I0Itec.zkclient.ZkClient -import kafka.utils.{Json, ZKGroupDirs, ZkUtils, Logging} +import kafka.utils.{Json, ZKGroupDirs, ZkUtils, Logging, Utils} import kafka.common.KafkaException private[kafka] trait TopicCount { @@ -125,7 +125,7 @@ private[kafka] class WildcardTopicCount(zkClient: ZkClient, makeConsumerThreadIdsPerTopic(consumerIdString, Map(wildcardTopics.map((_, numStreams)): _*)) } - def getTopicCountMap = Map(topicFilter.regex -> numStreams) + def getTopicCountMap = Map(Utils.JSONEscapeString(topicFilter.regex) -> numStreams) def pattern: String = { topicFilter match { diff --git a/core/src/main/scala/kafka/utils/Utils.scala b/core/src/main/scala/kafka/utils/Utils.scala index a89b046..480d5f0 100644 --- a/core/src/main/scala/kafka/utils/Utils.scala +++ b/core/src/main/scala/kafka/utils/Utils.scala @@ -540,5 +540,29 @@ object Utils extends Logging { lock.unlock() } } - + + + //JSON strings need to be escaped based on ECMA-404 standard http://json.org + def JSONEscapeString (s : String) : String = { + s.map { + case '"' => "\\\"" + case '\\' => "\\\\" + case '/' => "\\/" + case '\b' => "\\b" + case '\f' => "\\f" + case '\n' => "\\n" + case '\r' => "\\r" + case '\t' => "\\t" + /* We'll unicode escape any control characters. These include: + * 0x0 -> 0x1f : ASCII Control (C0 Control Codes) + * 0x7f : ASCII DELETE + * 0x80 -> 0x9f : C1 Control Codes + * + * Per RFC4627, section 2.5, we're not technically required to + * encode the C1 codes, but we do to be safe. + */ + case c if ((c >= '\u0000' && c <= '\u001f') || (c >= '\u007f' && c <= '\u009f')) => "\\u%04x".format(c: Int) + case c => c + }.mkString + } } diff --git a/core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala b/core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala index cf2724b..6c4a8d3 100644 --- a/core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala +++ b/core/src/test/scala/unit/kafka/consumer/TopicFilterTest.scala @@ -42,6 +42,29 @@ class TopicFilterTest extends JUnitSuite { @Test def testBlacklists() { - val topicFilter1 = new Blacklist("black1") + val topicFilter1 = new Blacklist("notthis.+") + assertTrue(topicFilter1.isTopicAllowed("something")) + assertFalse(topicFilter1.isTopicAllowed("notthis_anything")) } + + @Test + def testWildcardTopicCountGetTopicCountMapEscapeJson() { + def getTopicCountMapKey(regex: String): String = { + val topicCount = new WildcardTopicCount(null, "consumerId", new Whitelist(regex), 1) + topicCount.getTopicCountMap.head._1 + } + //lets make sure that the JSON strings are escaping as we expect + assertEquals("-\\\"-", getTopicCountMapKey("-\"-")) + assertEquals("-\\\\-", getTopicCountMapKey("-\\-")) + assertEquals("-\\/-", getTopicCountMapKey("-/-")) + assertEquals("-\\\\b-", getTopicCountMapKey("-\\b-")) + assertEquals("-\\\\f-", getTopicCountMapKey("-\\f-")) + assertEquals("-\\\\n-", getTopicCountMapKey("-\\n-")) + assertEquals("-\\\\r-", getTopicCountMapKey("-\\r-")) + assertEquals("-\\\\t-", getTopicCountMapKey("-\\t-")) + assertEquals("-\\\\u0000-", getTopicCountMapKey("-\\u0000-")) + assertEquals("-\\\\u001f-", getTopicCountMapKey("-\\u001f-")) + assertEquals("-\\\\u007f-", getTopicCountMapKey("-\\u007f-")) + assertEquals("-\\\\u009f-", getTopicCountMapKey("-\\u009f-")) + } } \ No newline at end of file