From 1f328e4d9ca808d172ba06abff460033269bbe32 Mon Sep 17 00:00:00 2001 From: Matt Christiansen Date: Fri, 5 Apr 2013 12:12:17 -0700 Subject: [PATCH 1/1] KAFKA-854 Upgrade Deps to latest --- .../java/kafka/bridge/pig/AvroKafkaStorage.java | 9 +++++---- core/build.sbt | 6 +++--- project/Build.scala | 22 +++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java b/contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java index d24a85a..66462c6 100644 --- a/contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java +++ b/contrib/hadoop-producer/src/main/java/kafka/bridge/pig/AvroKafkaStorage.java @@ -21,8 +21,9 @@ import java.io.IOException; import java.io.OutputStream; import kafka.bridge.hadoop.KafkaOutputFormat; import kafka.bridge.hadoop.KafkaRecordWriter; -import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.Schema; import org.apache.avro.io.Encoder; +import org.apache.avro.io.EncoderFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.OutputFormat; @@ -36,14 +37,14 @@ import org.apache.pig.piggybank.storage.avro.PigSchema2Avro; public class AvroKafkaStorage extends StoreFunc { protected KafkaRecordWriter writer; - protected org.apache.avro.Schema avroSchema; + protected Schema avroSchema; protected PigAvroDatumWriter datumWriter; protected Encoder encoder; protected ByteArrayOutputStream os; public AvroKafkaStorage(String schema) { - this.avroSchema = org.apache.avro.Schema.parse(schema); + this.avroSchema = new Schema.Parser().parse(schema); } @Override @@ -74,7 +75,7 @@ public class AvroKafkaStorage extends StoreFunc this.writer = (KafkaRecordWriter) writer; this.datumWriter = new PigAvroDatumWriter(this.avroSchema); this.os = new ByteArrayOutputStream(); - this.encoder = new BinaryEncoder(this.os); + this.encoder = EncoderFactory.get().binaryEncoder(os, null); } @Override diff --git a/core/build.sbt b/core/build.sbt index 211aaf9..8193161 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -10,11 +10,11 @@ resolvers ++= Seq( libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _ ) libraryDependencies ++= Seq( - "org.apache.zookeeper" % "zookeeper" % "3.3.4", + "org.apache.zookeeper" % "zookeeper" % "3.3.6", "com.github.sgroschupf" % "zkclient" % "0.1", "org.xerial.snappy" % "snappy-java" % "1.0.4.1", - "org.easymock" % "easymock" % "3.0" % "test", - "junit" % "junit" % "4.1" % "test" + "org.easymock" % "easymock" % "3.1" % "test", + "junit" % "junit" % "4.11" % "test" ) libraryDependencies <<= (scalaVersion, libraryDependencies) { (sv, deps) => diff --git a/project/Build.scala b/project/Build.scala index 4bbdfee..3237701 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -32,9 +32,9 @@ object KafkaBuild extends Build { javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.5"), parallelExecution in Test := false, // Prevent tests from overrunning each other libraryDependencies ++= Seq( - "log4j" % "log4j" % "1.2.15", + "log4j" % "log4j" % "1.2.17", "net.sf.jopt-simple" % "jopt-simple" % "3.2", - "org.slf4j" % "slf4j-simple" % "1.6.4" + "org.slf4j" % "slf4j-simple" % "1.7.3" ), // The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required // some dependencies on various sun and javax packages. @@ -44,7 +44,7 @@ object KafkaBuild extends Build { - + @@ -54,12 +54,12 @@ object KafkaBuild extends Build { val hadoopSettings = Seq( javacOptions ++= Seq("-Xlint:deprecation"), libraryDependencies ++= Seq( - "org.apache.avro" % "avro" % "1.4.0", - "org.apache.pig" % "pig" % "0.8.0", - "commons-logging" % "commons-logging" % "1.0.4", - "org.codehaus.jackson" % "jackson-core-asl" % "1.5.5", - "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5", - "org.apache.hadoop" % "hadoop-core" % "0.20.2" + "org.apache.avro" % "avro" % "1.7.4", + "org.apache.pig" % "pig" % "0.11.0", + "commons-logging" % "commons-logging" % "1.1.1", + "org.codehaus.jackson" % "jackson-core-asl" % "1.9.12", + "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.12", + "org.apache.hadoop" % "hadoop-core" % "1.1.2" ), ivyXML := @@ -69,10 +69,10 @@ object KafkaBuild extends Build { - + - + -- 1.8.2