Index: serde/if/test/complexpb.proto =================================================================== --- serde/if/test/complexpb.proto (revision 0) +++ serde/if/test/complexpb.proto (revision 0) @@ -0,0 +1,15 @@ +package org.apache.hadoop.hive.serde2.proto.test; + +message IntString { + optional int32 myint = 1; + optional string myString = 2; + optional int32 underscore_int = 3; +}; + +message Complex { + optional int32 aint = 1; + optional string aString = 2; + repeated int32 lint = 3; + repeated string lString = 4; + repeated IntString lintString = 5; +}; Index: serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java =================================================================== --- serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java (revision 0) +++ serde/src/gen-java/org/apache/hadoop/hive/serde2/proto/test/Complexpb.java (revision 0) @@ -0,0 +1,118 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: complexpb.proto + +package org.apache.hadoop.hive.serde2.proto.test; + +public final class Complexpb { + private Complexpb() {} + + public static final class IntString { + public IntString(int myInt, String myString, int underscoreInt) { + this.myint_ = myInt; + this.hasMyint = true; + this.myString_ = myString; + this.hasMyString = true; + this.underscoreInt_ = underscoreInt; + this.hasUnderscoreInt = true; + } + + // optional int32 myint = 1; + public static final int MYINT_FIELD_NUMBER = 1; + private final boolean hasMyint; + private int myint_ = 0; + public boolean hasMyint() { return hasMyint; } + public int getMyint() { return myint_; } + + // optional string myString = 2; + public static final int MYSTRING_FIELD_NUMBER = 2; + private final boolean hasMyString; + private java.lang.String myString_ = ""; + public boolean hasMyString() { return hasMyString; } + public java.lang.String getMyString() { return myString_; } + + // optional int32 underscore_int = 3; + public static final int UNDERSCORE_INT_FIELD_NUMBER = 3; + private final boolean hasUnderscoreInt; + private int underscoreInt_ = 0; + public boolean hasUnderscoreInt() { return hasUnderscoreInt; } + public int getUnderscoreInt() { return underscoreInt_; } + + public final boolean isInitialized() { + return true; + } + + } + + public static final class Complex { + // Use Complex.newBuilder() to construct. + public Complex(int aint, String aString, java.util.List lint, java.util.List lString, java.util.List lintString) { + this.aint_ = aint; + this.hasAint = true; + this.aString_ = aString; + this.hasAString = true; + this.lint_ = lint; + this.lString_ = lString; + this.lintString_ = lintString; + } + + // optional int32 aint = 1; + public static final int AINT_FIELD_NUMBER = 1; + private final boolean hasAint; + private int aint_ = 0; + public boolean hasAint() { return hasAint; } + public int getAint() { return aint_; } + + // optional string aString = 2; + public static final int ASTRING_FIELD_NUMBER = 2; + private final boolean hasAString; + private java.lang.String aString_ = ""; + public boolean hasAString() { return hasAString; } + public java.lang.String getAString() { return aString_; } + + // repeated int32 lint = 3; + public static final int LINT_FIELD_NUMBER = 3; + private java.util.List lint_ = + java.util.Collections.emptyList(); + public java.util.List getLintList() { + return lint_; + } + public int getLintCount() { return lint_.size(); } + public int getLint(int index) { + return lint_.get(index); + } + + // repeated string lString = 4; + public static final int LSTRING_FIELD_NUMBER = 4; + private java.util.List lString_ = + java.util.Collections.emptyList(); + public java.util.List getLStringList() { + return lString_; + } + public int getLStringCount() { return lString_.size(); } + public java.lang.String getLString(int index) { + return lString_.get(index); + } + + // repeated .org.apache.hadoop.hive.serde2.proto.test.IntString lintString = 5; + public static final int LINTSTRING_FIELD_NUMBER = 5; + private java.util.List lintString_ = + java.util.Collections.emptyList(); + public java.util.List getLintStringList() { + return lintString_; + } + public int getLintStringCount() { return lintString_.size(); } + public org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString getLintString(int index) { + return lintString_.get(index); + } + + private void initFields() { + } + public final boolean isInitialized() { + return true; + } + + // @@protoc_insertion_point(class_scope:org.apache.hadoop.hive.serde2.proto.test.Complex) + } + + // @@protoc_insertion_point(outer_class_scope) +} Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (revision 995077) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (working copy) @@ -23,6 +23,7 @@ import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,16 +49,16 @@ /** * ObjectInspectorOptions describes what ObjectInspector to use. JAVA is to * use pure JAVA reflection. THRIFT is to use JAVA reflection and filter out - * __isset fields. New ObjectInspectorOptions can be added here when - * available. - * + * __isset fields, PROTOCOL_BUFFERS filters out has*. + * New ObjectInspectorOptions can be added here when available. + * * We choose to use a single HashMap objectInspectorCache to cache all * situations for efficiency and code simplicity. And we don't expect a case * that a user need to create 2 or more different types of ObjectInspectors * for the same Java type. */ public enum ObjectInspectorOptions { - JAVA, THRIFT + JAVA, THRIFT, PROTOCOL_BUFFERS }; private static HashMap objectInspectorCache = new HashMap(); @@ -69,16 +70,37 @@ oi = getReflectionObjectInspectorNoCache(t, options); objectInspectorCache.put(t, oi); } - if ((options.equals(ObjectInspectorOptions.JAVA) && oi.getClass().equals( - ThriftStructObjectInspector.class)) - || (options.equals(ObjectInspectorOptions.THRIFT) && oi.getClass() - .equals(ReflectionStructObjectInspector.class))) { - throw new RuntimeException( - "Cannot call getObjectInspectorByReflection with both JAVA and THRIFT !"); - } + verifyObjectInspector(options, oi, ObjectInspectorOptions.JAVA, new Class[]{ThriftStructObjectInspector.class, + ProtocolBuffersStructObjectInspector.class}); + verifyObjectInspector(options, oi, ObjectInspectorOptions.THRIFT, new Class[]{ReflectionStructObjectInspector.class, + ProtocolBuffersStructObjectInspector.class}); + verifyObjectInspector(options, oi, ObjectInspectorOptions.PROTOCOL_BUFFERS, new Class[]{ThriftStructObjectInspector.class, + ReflectionStructObjectInspector.class}); + return oi; } + /** + * Verify that we don't have an unexpected type of object inspector. + * @param option The option to verify + * @param oi The ObjectInspector to verify + * @param checkOption We're only interested in this option type + * @param classes ObjectInspector should not be of these types + */ + private static void verifyObjectInspector(ObjectInspectorOptions option, ObjectInspector oi, + ObjectInspectorOptions checkOption, Class[] classes) { + + if (option.equals(checkOption)) { + for (Class checkClass : classes) { + if (oi.getClass().equals(checkClass)) { + throw new RuntimeException( + "Cannot call getObjectInspectorByReflection with more then one of " + + Arrays.toString(ObjectInspectorOptions.values()) + "!"); + } + } + } + } + private static ObjectInspector getReflectionObjectInspectorNoCache(Type t, ObjectInspectorOptions options) { if (t instanceof GenericArrayType) { @@ -147,6 +169,9 @@ case THRIFT: oi = new ThriftStructObjectInspector(); break; + case PROTOCOL_BUFFERS: + oi = new ProtocolBuffersStructObjectInspector(); + break; default: throw new RuntimeException(ObjectInspectorFactory.class.getName() + ": internal error."); Index: serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java =================================================================== --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java (revision 0) +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ProtocolBuffersStructObjectInspector.java (revision 0) @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.objectinspector; + +/** + * + * Always use the ObjectInspectorFactory to create new ObjectInspector objects, + * instead of directly creating an instance of this class. + */ +class ProtocolBuffersStructObjectInspector extends ReflectionStructObjectInspector { + + @Override + public boolean shouldIgnoreField(String name) { + return name.startsWith("has"); + } + +} Index: serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java =================================================================== --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java (revision 0) +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java (revision 0) @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.objectinspector; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.proto.test.Complexpb.Complex; +import org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString; + +/** + * TestProtocolBuffersObjectInspectors. + * + */ +public class TestProtocolBuffersObjectInspectors extends TestCase { + + public void testProtocolBuffersObjectInspectors() throws Throwable { + + try { + ObjectInspector oi1 = ObjectInspectorFactory + .getReflectionObjectInspector(Complex.class, + ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS); + ObjectInspector oi2 = ObjectInspectorFactory + .getReflectionObjectInspector(Complex.class, + ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS); + assertEquals(oi1, oi2); + + // metadata + assertEquals(Category.STRUCT, oi1.getCategory()); + StructObjectInspector soi = (StructObjectInspector) oi1; + List fields = soi.getAllStructFieldRefs(); + assertEquals(5, fields.size()); + assertEquals(fields.get(0), soi.getStructFieldRef("aint_")); + + // null + for (int i = 0; i < fields.size(); i++) { + assertNull(soi.getStructFieldData(null, fields.get(i))); + } + + // real object + List c2 = Arrays.asList(new Integer[] {1, 2, 3}); + List c3 = Arrays.asList(new String[] {"one", "two"}); + List c4 = new ArrayList(); + Complex c = new Complex(1, "test", c2, c3, c4); + + assertEquals(1, soi.getStructFieldData(c, fields.get(0))); + assertEquals("test", soi.getStructFieldData(c, fields.get(1))); + assertEquals(c2, soi.getStructFieldData(c, fields.get(2))); + assertEquals(c3, soi.getStructFieldData(c, fields.get(3))); + assertEquals(c4, soi.getStructFieldData(c, fields.get(4))); + ArrayList cfields = new ArrayList(); + for (int i = 0; i < 5; i++) { + cfields.add(soi.getStructFieldData(c, fields.get(i))); + } + assertEquals(cfields, soi.getStructFieldsDataAsList(c)); + + // sub fields + assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, + fields.get(0).getFieldObjectInspector()); + assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, + fields.get(1).getFieldObjectInspector()); + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), + fields.get(2).getFieldObjectInspector()); + assertEquals( + ObjectInspectorFactory + .getStandardListObjectInspector(PrimitiveObjectInspectorFactory + .javaStringObjectInspector), + fields.get(3).getFieldObjectInspector()); + assertEquals(ObjectInspectorFactory + .getStandardListObjectInspector(ObjectInspectorFactory + .getReflectionObjectInspector(IntString.class, + ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS)), + fields.get(4).getFieldObjectInspector()); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } +}