Index: ivy.xml =================================================================== --- ivy.xml (revision 794132) +++ ivy.xml (working copy) @@ -269,6 +269,14 @@ rev="${slf4j-log4j12.version}" conf="common->master"> + + Index: lib/avro-1.0.0.jar =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lib/avro-1.0.0.jar ___________________________________________________________________ Name: svn:mime-type + application/octet-stream Index: src/test/core/org/apache/hadoop/io/serializer/TestWritableSerialization.java =================================================================== --- src/test/core/org/apache/hadoop/io/serializer/TestWritableSerialization.java (revision 794132) +++ src/test/core/org/apache/hadoop/io/serializer/TestWritableSerialization.java (working copy) @@ -23,12 +23,9 @@ import junit.framework.TestCase; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.TestGenericWritable.Baz; import org.apache.hadoop.io.TestGenericWritable.FooGenericWritable; -import org.apache.hadoop.util.GenericsUtil; public class TestWritableSerialization extends TestCase { @@ -41,7 +38,8 @@ public void testWritableSerialization() throws Exception { Text before = new Text("test writable"); - testSerialization(conf, before); + Text after = SerializationTestUtil.testSerialization(conf, before); + assertEquals(before, after); } @@ -56,40 +54,8 @@ generic.setConf(conf); Baz baz = new Baz(); generic.set(baz); - Baz result = testSerialization(conf, baz); + Baz result = SerializationTestUtil.testSerialization(conf, baz); + assertEquals(baz, result); assertNotNull(result.getConf()); } - - /** - * A utility that tests serialization/deserialization. - * @param the class of the item - * @param conf configuration to use, "io.serializations" is read to - * determine the serialization - * @param before item to (de)serialize - * @return deserialized item - */ - public static K testSerialization(Configuration conf, K before) - throws Exception { - - SerializationFactory factory = new SerializationFactory(conf); - Serializer serializer - = factory.getSerializer(GenericsUtil.getClass(before)); - Deserializer deserializer - = factory.getDeserializer(GenericsUtil.getClass(before)); - - DataOutputBuffer out = new DataOutputBuffer(); - serializer.open(out); - serializer.serialize(before); - serializer.close(); - - DataInputBuffer in = new DataInputBuffer(); - in.reset(out.getData(), out.getLength()); - deserializer.open(in); - K after = deserializer.deserialize(null); - deserializer.close(); - - assertEquals(before, after); - return after; - } - } Index: src/test/core/org/apache/hadoop/io/serializer/avro/avroRecord.avsc =================================================================== --- src/test/core/org/apache/hadoop/io/serializer/avro/avroRecord.avsc (revision 0) +++ src/test/core/org/apache/hadoop/io/serializer/avro/avroRecord.avsc (revision 0) @@ -0,0 +1,6 @@ +{"type": "record", "name":"AvroRecord", + "namespace": "org.apache.hadoop.io.serializer.avro", + "fields": [ + {"name": "intField", "type": "int"} + ] +} Index: src/test/core/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java =================================================================== --- src/test/core/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java (revision 0) +++ src/test/core/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java (revision 0) @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +import junit.framework.TestCase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.serializer.SerializationTestUtil; + +public class TestAvroSerialization extends TestCase { + + private static final Configuration conf = new Configuration(); + + static { + conf.set("io.serializations" + , "org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,"+ + "org.apache.hadoop.io.serializer.avro.AvroReflectSerialization"); + } + + public void testSpecific() throws Exception { + AvroRecord before = new AvroRecord(); + before.intField = 5; + AvroRecord after = SerializationTestUtil.testSerialization(conf, before); + assertEquals(before, after); + } + + public void testReflectPkg() throws Exception { + Record before = new Record(); + before.x = 10; + conf.set(AvroReflectSerialization.AVRO_REFLECT_PACKAGES, + before.getClass().getPackage().getName()); + Record after = SerializationTestUtil.testSerialization(conf, before); + assertEquals(before, after); + } + + public void testReflectInnerClass() throws Exception { + InnerRecord before = new InnerRecord(); + before.x = 10; + conf.set(AvroReflectSerialization.AVRO_REFLECT_PACKAGES, + before.getClass().getPackage().getName()); + InnerRecord after = SerializationTestUtil.testSerialization(conf, before); + assertEquals(before, after); + } + + public void testReflect() throws Exception { + RefSerializable before = new RefSerializable(); + before.x = 10; + RefSerializable after = + SerializationTestUtil.testSerialization(conf, before); + assertEquals(before, after); + } + + public static class InnerRecord { + public int x = 7; + + public int hashCode() { + return x; + } + + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final InnerRecord other = (InnerRecord) obj; + if (x != other.x) + return false; + return true; + } + } + + public static class RefSerializable implements AvroReflectSerializable { + public int x = 7; + + public int hashCode() { + return x; + } + + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final RefSerializable other = (RefSerializable) obj; + if (x != other.x) + return false; + return true; + } + } +} Index: src/test/core/org/apache/hadoop/io/serializer/avro/Record.java =================================================================== --- src/test/core/org/apache/hadoop/io/serializer/avro/Record.java (revision 0) +++ src/test/core/org/apache/hadoop/io/serializer/avro/Record.java (revision 0) @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +public class Record { + public int x = 7; + + public int hashCode() { + return x; + } + + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final Record other = (Record) obj; + if (x != other.x) + return false; + return true; + } +} \ No newline at end of file Index: src/test/core/org/apache/hadoop/io/serializer/SerializationTestUtil.java =================================================================== --- src/test/core/org/apache/hadoop/io/serializer/SerializationTestUtil.java (revision 0) +++ src/test/core/org/apache/hadoop/io/serializer/SerializationTestUtil.java (revision 0) @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.serializer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.util.GenericsUtil; + +public class SerializationTestUtil { + + /** + * A utility that tests serialization/deserialization. + * @param the class of the item + * @param conf configuration to use, "io.serializations" is read to + * determine the serialization + * @param before item to (de)serialize + * @return deserialized item + */ + public static K testSerialization(Configuration conf, K before) + throws Exception { + + SerializationFactory factory = new SerializationFactory(conf); + Serializer serializer + = factory.getSerializer(GenericsUtil.getClass(before)); + Deserializer deserializer + = factory.getDeserializer(GenericsUtil.getClass(before)); + + DataOutputBuffer out = new DataOutputBuffer(); + serializer.open(out); + serializer.serialize(before); + serializer.close(); + + DataInputBuffer in = new DataInputBuffer(); + in.reset(out.getData(), out.getLength()); + deserializer.open(in); + K after = deserializer.deserialize(null); + deserializer.close(); + return after; + } + +} Index: src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerializable.java =================================================================== --- src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerializable.java (revision 0) +++ src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerializable.java (revision 0) @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +/** + * Tag interface for Avro reflect serializable classes. + * + */ +public interface AvroReflectSerializable { + +} Index: src/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java =================================================================== --- src/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java (revision 0) +++ src/java/org/apache/hadoop/io/serializer/avro/AvroSerialization.java (revision 0) @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.apache.avro.Schema; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.serializer.Deserializer; +import org.apache.hadoop.io.serializer.Serialization; +import org.apache.hadoop.io.serializer.Serializer; + +/** + * Serialization for Avro classes. + * + */ +abstract class AvroSerialization extends Configured + implements Serialization{ + + public Deserializer getDeserializer(Class c) { + return new AvroDeserializer(c); + } + + public Serializer getSerializer(Class c) { + return new AvroSerializer(c); + } + + protected abstract Schema getSchema(T t); + + protected abstract DatumWriter getWriter(Class clazz); + + protected abstract DatumReader getReader(Class clazz); + + class AvroSerializer implements Serializer { + + private DatumWriter writer; + private BinaryEncoder encoder; + protected Class clazz; + + AvroSerializer(Class clazz) { + writer = getWriter(clazz); + } + + public void close() throws IOException { + } + + public void open(OutputStream out) throws IOException { + encoder = new BinaryEncoder(out); + } + + public void serialize(T t) throws IOException { + writer.setSchema(getSchema(t)); + writer.write(t, encoder); + } + + } + + class AvroDeserializer implements Deserializer { + + private DatumReader reader; + private BinaryDecoder decoder; + + AvroDeserializer(Class clazz) { + this.reader = getReader(clazz); + } + + public void close() throws IOException { + } + + public T deserialize(T t) throws IOException { + return reader.read(t, decoder); + } + + public void open(InputStream in) throws IOException { + decoder = new BinaryDecoder(in); + } + + } + +} Index: src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java =================================================================== --- src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java (revision 0) +++ src/java/org/apache/hadoop/io/serializer/avro/AvroReflectSerialization.java (revision 0) @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.reflect.ReflectData; +import org.apache.avro.reflect.ReflectDatumReader; +import org.apache.avro.reflect.ReflectDatumWriter; + +/** + * Serialization for Avro Reflect classes. + * + */ +@SuppressWarnings("unchecked") +public class AvroReflectSerialization extends AvroSerialization{ + + public static final String AVRO_REFLECT_PACKAGES = "avro.reflect.pkgs"; + + private Set packages; + + public synchronized boolean accept(Class c) { + if (packages == null) { + getPackages(); + } + return AvroReflectSerializable.class.isAssignableFrom(c) || + packages.contains(c.getPackage().getName()); + } + + private void getPackages() { + String[] pkgList = getConf().getStrings(AVRO_REFLECT_PACKAGES); + packages = new HashSet(); + if (pkgList != null) { + for (String pkg : pkgList) { + packages.add(pkg.trim()); + } + } + } + + protected DatumReader getReader(Class clazz) { + try { + String prefix = + ((clazz.getEnclosingClass() == null + || "null".equals(clazz.getEnclosingClass().getName())) ? + clazz.getPackage().getName() + "." + : (clazz.getEnclosingClass().getName() + "$")); + return new ReflectDatumReader(ReflectData.getSchema(clazz), prefix); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected Schema getSchema(Object t) { + return ReflectData.getSchema(t.getClass()); + } + + protected DatumWriter getWriter(Class clazz) { + return new ReflectDatumWriter(); + } + +} Index: src/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java =================================================================== --- src/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java (revision 0) +++ src/java/org/apache/hadoop/io/serializer/avro/AvroSpecificSerialization.java (revision 0) @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.serializer.avro; + +import org.apache.avro.Schema; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.specific.SpecificDatumReader; +import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.avro.specific.SpecificRecord; + +/** + * Serialization for Avro Specific classes. + * + */ +@SuppressWarnings("unchecked") +public class AvroSpecificSerialization + extends AvroSerialization{ + + public boolean accept(Class c) { + return SpecificRecord.class.isAssignableFrom(c); + } + + protected DatumReader getReader(Class clazz) { + try { + return new SpecificDatumReader(clazz.newInstance().schema()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected Schema getSchema(SpecificRecord t) { + return t.schema(); + } + + protected DatumWriter getWriter(Class clazz) { + return new SpecificDatumWriter(); + } + +} Index: src/java/org/apache/hadoop/io/serializer/avro/package.html =================================================================== --- src/java/org/apache/hadoop/io/serializer/avro/package.html (revision 0) +++ src/java/org/apache/hadoop/io/serializer/avro/package.html (revision 0) @@ -0,0 +1,27 @@ + + + + + + +

+This package provides Avro serialization in Hadoop. +

+ + + Index: build.xml =================================================================== --- build.xml (revision 794132) +++ build.xml (working copy) @@ -169,6 +169,9 @@ + + + @@ -416,11 +419,22 @@ includes="**/*.jr" /> - + + + + + + + + + + + + - +