Kafka
  1. Kafka
  2. KAFKA-898

Add a KafkaMetricsReporter that wraps Librato's reporter

    Details

    • Type: Improvement Improvement
    • Status: Closed
    • Priority: Major Major
    • Resolution: Won't Fix
    • Affects Version/s: None
    • Fix Version/s: None
    • Component/s: None
    • Labels:
      None

      Activity

      Hide
      Scott Clasen added a comment -

      Subject: [PATCH] add optional librato metrics reporter


      config/server.properties | 9 ++++++++
      core/build.sbt | 1 +
      .../metrics/KafkaLibratoMetricsReporter.scala | 26 ++++++++++++++++++++++
      3 files changed, 36 insertions
      create mode 100644 core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala

      diff --git a/config/server.properties b/config/server.properties
      index bc6a521..a48951c 100644
      — a/config/server.properties
      +++ b/config/server.properties
      @@ -104,6 +104,7 @@ log.cleanup.interval.mins=1

      1. root directory for all kafka znodes.
        zookeeper.connect=localhost:2181

      +

      1. Timeout in ms for connecting to zookeeper
        zookeeper.connection.timeout.ms=1000000

      @@ -114,3 +115,11 @@ kafka.csv.metrics.dir=/tmp/kafka_metrics

      1. Disable csv reporting by default.
        kafka.csv.metrics.reporter.enabled=false

      +#kafka.metrics.reporters=kafka.metrics.KafkaLibratoMetricsReporter
      +#kafka.librato.metrics.reporter.enabled=true
      +kafka.librato.metrics.reporter.account=your@librato.account.com
      +kafka.librato.metrics.reporter.token=somelibratoapikey
      +kafka.librato.metrics.reporter.prefix=testing123
      +kafka.librato.metrics.reporter.source=testingsource
      +kafka.librato.metrics.reporter.interval.secs=5
      +
      diff --git a/core/build.sbt b/core/build.sbt
      index 405ea55..e12fa4b 100644
      — a/core/build.sbt
      +++ b/core/build.sbt
      @@ -16,6 +16,7 @@ libraryDependencies ++= Seq(
      "org.xerial.snappy" % "snappy-java" % "1.0.4.1",
      "com.yammer.metrics" % "metrics-core" % "2.2.0",
      "com.yammer.metrics" % "metrics-annotation" % "2.2.0",
      + "com.librato.metrics" % "metrics-librato" % "2.2.0.0",
      "org.easymock" % "easymock" % "3.0" % "test",
      "junit" % "junit" % "4.1" % "test"
      )
      diff --git a/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala b/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala
      new file mode 100644
      index 0000000..5070b98
      — /dev/null
      +++ b/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala
      @@ -0,0 +1,26 @@
      +package kafka.metrics
      +
      +import kafka.utils.VerifiableProperties
      +import com.librato.metrics.

      {APIUtil, LibratoReporter}

      +import java.util.concurrent.TimeUnit
      +
      +
      +class KafkaLibratoMetricsReporter extends KafkaMetricsReporter {
      + def init(props: VerifiableProperties) {
      + val enabled = props.getBoolean("kafka.librato.metrics.reporter.enabled", false)
      + if(enabled)

      { + val account = props.getString("kafka.librato.metrics.reporter.account") + val token = props.getString("kafka.librato.metrics.reporter.token") + val source = props.getString("kafka.librato.metrics.reporter.source") + val prefix = Option(props.getString("kafka.librato.metrics.reporter.prefix", null)) + val interval = props.getInt("kafka.librato.metrics.reporter.interval.secs",30) + var builder = LibratoReporter.builder(account, token, source) + prefix.foreach(p => builder = builder.setSanitizer(new PrefixingSanitizer(p))) + LibratoReporter.enable(builder, interval, TimeUnit.SECONDS) + }

      + }
      +}
      +
      +class PrefixingSanitizer(prefix:String) extends APIUtil.Sanitizer

      { + def apply(name: String): String = prefix + "." + name +}


      1.8.0.1

      Show
      Scott Clasen added a comment - Subject: [PATCH] add optional librato metrics reporter — config/server.properties | 9 ++++++++ core/build.sbt | 1 + .../metrics/KafkaLibratoMetricsReporter.scala | 26 ++++++++++++++++++++++ 3 files changed, 36 insertions create mode 100644 core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala diff --git a/config/server.properties b/config/server.properties index bc6a521..a48951c 100644 — a/config/server.properties +++ b/config/server.properties @@ -104,6 +104,7 @@ log.cleanup.interval.mins=1 root directory for all kafka znodes. zookeeper.connect=localhost:2181 + Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=1000000 @@ -114,3 +115,11 @@ kafka.csv.metrics.dir=/tmp/kafka_metrics Disable csv reporting by default. kafka.csv.metrics.reporter.enabled=false +#kafka.metrics.reporters=kafka.metrics.KafkaLibratoMetricsReporter +#kafka.librato.metrics.reporter.enabled=true +kafka.librato.metrics.reporter.account=your@librato.account.com +kafka.librato.metrics.reporter.token=somelibratoapikey +kafka.librato.metrics.reporter.prefix=testing123 +kafka.librato.metrics.reporter.source=testingsource +kafka.librato.metrics.reporter.interval.secs=5 + diff --git a/core/build.sbt b/core/build.sbt index 405ea55..e12fa4b 100644 — a/core/build.sbt +++ b/core/build.sbt @@ -16,6 +16,7 @@ libraryDependencies ++= Seq( "org.xerial.snappy" % "snappy-java" % "1.0.4.1", "com.yammer.metrics" % "metrics-core" % "2.2.0", "com.yammer.metrics" % "metrics-annotation" % "2.2.0", + "com.librato.metrics" % "metrics-librato" % "2.2.0.0", "org.easymock" % "easymock" % "3.0" % "test", "junit" % "junit" % "4.1" % "test" ) diff --git a/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala b/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala new file mode 100644 index 0000000..5070b98 — /dev/null +++ b/core/src/main/scala/kafka/metrics/KafkaLibratoMetricsReporter.scala @@ -0,0 +1,26 @@ +package kafka.metrics + +import kafka.utils.VerifiableProperties +import com.librato.metrics. {APIUtil, LibratoReporter} +import java.util.concurrent.TimeUnit + + +class KafkaLibratoMetricsReporter extends KafkaMetricsReporter { + def init(props: VerifiableProperties) { + val enabled = props.getBoolean("kafka.librato.metrics.reporter.enabled", false) + if(enabled) { + val account = props.getString("kafka.librato.metrics.reporter.account") + val token = props.getString("kafka.librato.metrics.reporter.token") + val source = props.getString("kafka.librato.metrics.reporter.source") + val prefix = Option(props.getString("kafka.librato.metrics.reporter.prefix", null)) + val interval = props.getInt("kafka.librato.metrics.reporter.interval.secs",30) + var builder = LibratoReporter.builder(account, token, source) + prefix.foreach(p => builder = builder.setSanitizer(new PrefixingSanitizer(p))) + LibratoReporter.enable(builder, interval, TimeUnit.SECONDS) + } + } +} + +class PrefixingSanitizer(prefix:String) extends APIUtil.Sanitizer { + def apply(name: String): String = prefix + "." + name +} – 1.8.0.1
      Hide
      Jay Kreps added a comment -

      Hey Scott, thanks for the patch!

      I think it would be better for us to avoid trying to include reporters for each possible metrics collection system. This should be easy enough for people to add into their deployment, and I am concerned about accumulating dependencies. Any objection to that strategy?

      Show
      Jay Kreps added a comment - Hey Scott, thanks for the patch! I think it would be better for us to avoid trying to include reporters for each possible metrics collection system. This should be easy enough for people to add into their deployment, and I am concerned about accumulating dependencies. Any objection to that strategy?
      Hide
      Scott Clasen added a comment -

      Hey Jay

      Makes sense.

      I had seen a bunch of questions on the mailing list around "How do I monitor Kafka/How do I see metrics from kafka" and figured this would be a super easy way for folks to start looking at metrics...and since Coda is an advisor at Librato, thought this might be a more 'preferred' reporter...But perhaps Librato themselves should publish this to maven central. Once Kafka is there...nudge nudge..

      Or alternatively is there a place in contrib that this can live that makes it not part of the core dependency chain but easy to use if desired?

      Cheers
      SC

      Show
      Scott Clasen added a comment - Hey Jay Makes sense. I had seen a bunch of questions on the mailing list around "How do I monitor Kafka/How do I see metrics from kafka" and figured this would be a super easy way for folks to start looking at metrics...and since Coda is an advisor at Librato, thought this might be a more 'preferred' reporter...But perhaps Librato themselves should publish this to maven central. Once Kafka is there...nudge nudge.. Or alternatively is there a place in contrib that this can live that makes it not part of the core dependency chain but easy to use if desired? Cheers SC
      Hide
      Jay Kreps added a comment -

      We can dump it in contrib, but I think what we have found is that contrib is a pretty bad place for plugins. It takes less time to make a github project then to interact on JIRA and that way the author can keep iterating on the code unencumbered. So I think the thinking on contrib is to instead either take it as a first-class part of the project that we maintain indefinitely or let it thrive elsewhere.

      Show
      Jay Kreps added a comment - We can dump it in contrib, but I think what we have found is that contrib is a pretty bad place for plugins. It takes less time to make a github project then to interact on JIRA and that way the author can keep iterating on the code unencumbered. So I think the thinking on contrib is to instead either take it as a first-class part of the project that we maintain indefinitely or let it thrive elsewhere.
      Hide
      Jay Kreps added a comment -

      Will be happier elsewhere.

      Show
      Jay Kreps added a comment - Will be happier elsewhere.
      Hide
      Scott Clasen added a comment -

      Cool ill see if the librato folks want it, if not ill publish something to maven central, once Kafka is there

      Show
      Scott Clasen added a comment - Cool ill see if the librato folks want it, if not ill publish something to maven central, once Kafka is there

        People

        • Assignee:
          Unassigned
          Reporter:
          Scott Clasen
        • Votes:
          0 Vote for this issue
          Watchers:
          3 Start watching this issue

          Dates

          • Created:
            Updated:
            Resolved:

            Development