diff --git ivy/libraries.properties ivy/libraries.properties index af856bd..4236df1 100644 --- ivy/libraries.properties +++ ivy/libraries.properties @@ -44,6 +44,7 @@ junit.version=3.8.1 ivy.version=2.1.0 log4j.version=1.2.15 maven-ant-tasks.version=2.1.0 +mockito-all.version=1.8.2 slf4j-api.version=1.6.1 slf4j-log4j12.version=1.6.1 thrift.version=0.5.0 diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index c1fa4e5..8bd26ca 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -823,21 +823,30 @@ public class MetaStoreUtils { // rules on how to recurse the ObjectInspector based on its type if (oi.getCategory() != Category.STRUCT) { str_fields.add(new FieldSchema(last_name, oi.getTypeName(), - "from deserializer")); + FROM_SERIALIZER)); } else { List fields = ((StructObjectInspector) oi) .getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { - String fieldName = fields.get(i).getFieldName(); - String fieldTypeName = fields.get(i).getFieldObjectInspector() - .getTypeName(); - str_fields.add(new FieldSchema(fieldName, fieldTypeName, - "from deserializer")); + StructField structField = fields.get(i); + String fieldName = structField.getFieldName(); + String fieldTypeName = structField.getFieldObjectInspector().getTypeName(); + String fieldComment = determineFieldComment(structField.getFieldComment()); + + str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment)); } } return str_fields; } + private static final String FROM_SERIALIZER = "from deserializer"; + private static String determineFieldComment(String comment) { + if(comment == null || comment.isEmpty()) + return FROM_SERIALIZER; + else + return comment; + } + /** * Convert TypeInfo to FieldSchema. */ diff --git serde/ivy.xml serde/ivy.xml index d6c836a..01c4d0c 100644 --- serde/ivy.xml +++ serde/ivy.xml @@ -28,5 +28,6 @@ + diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java index 4850601..8ef5be7 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java @@ -46,6 +46,16 @@ public final class LazyObjectInspectorFactory { List structFieldObjectInspectors, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { + return getLazySimpleStructObjectInspector(structFieldNames, + structFieldObjectInspectors, null, separator, nullSequence, + lastColumnTakesRest, escaped, escapeChar); + } + + public static LazySimpleStructObjectInspector getLazySimpleStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors, List structFieldComments, + byte separator, Text nullSequence, boolean lastColumnTakesRest, + boolean escaped,byte escapeChar) { ArrayList signature = new ArrayList(); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); @@ -54,12 +64,13 @@ public final class LazyObjectInspectorFactory { signature.add(Boolean.valueOf(lastColumnTakesRest)); signature.add(Boolean.valueOf(escaped)); signature.add(Byte.valueOf(escapeChar)); + if(structFieldComments != null) signature.add(structFieldComments); LazySimpleStructObjectInspector result = cachedLazySimpleStructObjectInspector .get(signature); if (result == null) { result = new LazySimpleStructObjectInspector(structFieldNames, - structFieldObjectInspectors, separator, nullSequence, - lastColumnTakesRest, escaped, escapeChar); + structFieldObjectInspectors, structFieldComments, separator, + nullSequence, lastColumnTakesRest, escaped, escapeChar); cachedLazySimpleStructObjectInspector.put(signature, result); } return result; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java index e2fa9db..08400f1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java @@ -49,6 +49,7 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { protected int fieldID; protected String fieldName; protected ObjectInspector fieldObjectInspector; + protected String fieldComment; public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) { @@ -57,6 +58,11 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { this.fieldObjectInspector = fieldObjectInspector; } + public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector, String fieldComment) { + this(fieldID, fieldName, fieldObjectInspector); + this.fieldComment = fieldComment; + } + public int getFieldID() { return fieldID; } @@ -69,6 +75,10 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { return fieldObjectInspector; } + public String getFieldComment() { + return fieldComment; + } + @Override public String toString() { return "" + fieldID + ":" + fieldName; @@ -95,15 +105,26 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { List structFieldObjectInspectors, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { - init(structFieldNames, structFieldObjectInspectors, separator, + init(structFieldNames, structFieldObjectInspectors, null, separator, nullSequence, lastColumnTakesRest, escaped, escapeChar); } + public LazySimpleStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments, byte separator, Text nullSequence, + boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { + init(structFieldNames, structFieldObjectInspectors, structFieldComments, + separator, nullSequence, lastColumnTakesRest, escaped, escapeChar); + } + protected void init(List structFieldNames, - List structFieldObjectInspectors, byte separator, + List structFieldObjectInspectors, + List structFieldComments, byte separator, Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) { assert (structFieldNames.size() == structFieldObjectInspectors.size()); + assert (structFieldComments == null || + structFieldNames.size() == structFieldComments.size()); this.separator = separator; this.nullSequence = nullSequence; @@ -114,7 +135,8 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { fields = new ArrayList(structFieldNames.size()); for (int i = 0; i < structFieldNames.size(); i++) { fields.add(new MyField(i, structFieldNames.get(i), - structFieldObjectInspectors.get(i))); + structFieldObjectInspectors.get(i), + structFieldComments == null ? null : structFieldComments.get(i))); } } @@ -131,7 +153,7 @@ public class LazySimpleStructObjectInspector extends StructObjectInspector { this.fields = new ArrayList(fields.size()); for (int i = 0; i < fields.size(); i++) { this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields - .get(i).getFieldObjectInspector())); + .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment())); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java index 2947e49..0675908 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryObjectInspectorFactory.java @@ -42,14 +42,22 @@ public final class LazyBinaryObjectInspectorFactory { public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector( List structFieldNames, List structFieldObjectInspectors) { - ArrayList signature = new ArrayList(); + return getLazyBinaryStructObjectInspector(structFieldNames, + structFieldObjectInspectors, null); + } + + public static LazyBinaryStructObjectInspector getLazyBinaryStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors, List structFieldComments) { + ArrayList signature = new ArrayList(3); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); + if(structFieldComments != null) signature.add(structFieldComments); LazyBinaryStructObjectInspector result = cachedLazyBinaryStructObjectInspector .get(signature); if (result == null) { result = new LazyBinaryStructObjectInspector(structFieldNames, - structFieldObjectInspectors); + structFieldObjectInspectors, structFieldComments); cachedLazyBinaryStructObjectInspector.put(signature, result); } return result; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java index 3d5408f..7141aac 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/objectinspector/LazyBinaryStructObjectInspector.java @@ -37,6 +37,12 @@ public class LazyBinaryStructObjectInspector extends super(structFieldNames, structFieldObjectInspectors); } + protected LazyBinaryStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments) { + super(structFieldNames, structFieldObjectInspectors, structFieldComments); + } + protected LazyBinaryStructObjectInspector(List fields) { super(fields); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java index 66f4f8d..c05727a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java @@ -45,6 +45,7 @@ class ColumnarStructObjectInspector extends StructObjectInspector { protected int fieldID; protected String fieldName; protected ObjectInspector fieldObjectInspector; + protected String fieldComment; public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) { @@ -53,6 +54,12 @@ class ColumnarStructObjectInspector extends StructObjectInspector { this.fieldObjectInspector = fieldObjectInspector; } + public MyField(int fieldID, String fieldName, + ObjectInspector fieldObjectInspector, String fieldComment) { + this(fieldID, fieldName, fieldObjectInspector); + this.fieldComment = fieldComment; + } + public int getFieldID() { return fieldID; } @@ -65,6 +72,9 @@ class ColumnarStructObjectInspector extends StructObjectInspector { return fieldObjectInspector; } + public String getFieldComment() { + return fieldComment; + } @Override public String toString() { return "" + fieldID + ":" + fieldName; @@ -85,19 +95,30 @@ class ColumnarStructObjectInspector extends StructObjectInspector { */ public ColumnarStructObjectInspector(List structFieldNames, List structFieldObjectInspectors, Text nullSequence) { - init(structFieldNames, structFieldObjectInspectors, nullSequence); + init(structFieldNames, structFieldObjectInspectors, null, nullSequence); + } + + public ColumnarStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments, Text nullSequence) { + init(structFieldNames, structFieldObjectInspectors, structFieldComments, + nullSequence); } protected void init(List structFieldNames, - List structFieldObjectInspectors, Text nullSequence) { + List structFieldObjectInspectors, + List structFieldComments, Text nullSequence) { assert (structFieldNames.size() == structFieldObjectInspectors.size()); + assert (structFieldComments == null || + (structFieldNames.size() == structFieldComments.size())); this.nullSequence = nullSequence; fields = new ArrayList(structFieldNames.size()); for (int i = 0; i < structFieldNames.size(); i++) { fields.add(new MyField(i, structFieldNames.get(i), - structFieldObjectInspectors.get(i))); + structFieldObjectInspectors.get(i), + structFieldComments == null ? null : structFieldComments.get(i))); } } @@ -112,7 +133,7 @@ class ColumnarStructObjectInspector extends StructObjectInspector { this.fields = new ArrayList(fields.size()); for (int i = 0; i < fields.size(); i++) { this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields - .get(i).getFieldObjectInspector())); + .get(i).getFieldObjectInspector(), fields.get(i).getFieldComment())); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java index bd42a0c..b8d3421 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.serde2.objectinspector; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -38,17 +39,33 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn public class MetadataListStructObjectInspector extends StandardStructObjectInspector { - static HashMap, MetadataListStructObjectInspector> cached = new HashMap, MetadataListStructObjectInspector>(); + static HashMap>, MetadataListStructObjectInspector> + cached = new HashMap>, MetadataListStructObjectInspector>(); // public static MetadataListStructObjectInspector getInstance(int fields) { // return getInstance(ObjectInspectorUtils.getIntegerArray(fields)); // } public static MetadataListStructObjectInspector getInstance( List columnNames) { + ArrayList> key = new ArrayList>(1); + key.add(columnNames); MetadataListStructObjectInspector result = cached.get(columnNames); if (result == null) { result = new MetadataListStructObjectInspector(columnNames); - cached.put(columnNames, result); + cached.put(key, result); + } + return result; + } + + public static MetadataListStructObjectInspector getInstance( + List columnNames, List columnComments) { + ArrayList> key = new ArrayList>(2); + Collections.addAll(key, columnNames, columnComments); + + MetadataListStructObjectInspector result = cached.get(key); + if (result == null) { + result = new MetadataListStructObjectInspector(columnNames, columnComments); + cached.put(key, result); } return result; } @@ -66,6 +83,12 @@ public class MetadataListStructObjectInspector extends super(columnNames, getFieldObjectInspectors(columnNames.size())); } + public MetadataListStructObjectInspector(List columnNames, + List columnComments) { + super(columnNames, getFieldObjectInspectors(columnNames.size()), + columnComments); + } + // Get col object out @Override public Object getStructFieldData(Object data, StructField fieldRef) { diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java index 90561a1..1268bc9 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java @@ -243,14 +243,20 @@ public final class ObjectInspectorFactory { public static StandardStructObjectInspector getStandardStructObjectInspector( List structFieldNames, List structFieldObjectInspectors) { - ArrayList> signature = new ArrayList>(); + return getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, null); + } + + public static StandardStructObjectInspector getStandardStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors, + List structComments) { + ArrayList> signature = new ArrayList>(3); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); - StandardStructObjectInspector result = cachedStandardStructObjectInspector - .get(signature); - if (result == null) { - result = new StandardStructObjectInspector(structFieldNames, - structFieldObjectInspectors); + if(structComments != null) signature.add(structComments); + StandardStructObjectInspector result = cachedStandardStructObjectInspector.get(signature); + if(result == null) { + result = new StandardStructObjectInspector(structFieldNames, structFieldObjectInspectors, structComments); cachedStandardStructObjectInspector.put(signature, result); } return result; @@ -274,15 +280,22 @@ public final class ObjectInspectorFactory { public static ColumnarStructObjectInspector getColumnarStructObjectInspector( List structFieldNames, List structFieldObjectInspectors, Text nullSequence) { - ArrayList signature = new ArrayList(); + return getColumnarStructObjectInspector(structFieldNames, structFieldObjectInspectors, null, nullSequence); + } + + public static ColumnarStructObjectInspector getColumnarStructObjectInspector( + List structFieldNames, + List structFieldObjectInspectors, List structFieldComments, Text nullSequence) { + ArrayList signature = new ArrayList(4); signature.add(structFieldNames); signature.add(structFieldObjectInspectors); + if(structFieldComments != null) signature.add(structFieldComments); signature.add(nullSequence.toString()); ColumnarStructObjectInspector result = cachedColumnarStructObjectInspector .get(signature); if (result == null) { result = new ColumnarStructObjectInspector(structFieldNames, - structFieldObjectInspectors, nullSequence); + structFieldObjectInspectors, structFieldComments, nullSequence); cachedColumnarStructObjectInspector.put(signature, result); } return result; diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java index 4a934c5..ff63163 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java @@ -58,6 +58,10 @@ public class ReflectionStructObjectInspector extends return fieldObjectInspector; } + public String getFieldComment() { + return null; + } + @Override public String toString() { return field.toString(); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java index 3b26e45..e89163c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java @@ -46,6 +46,7 @@ public class StandardStructObjectInspector extends protected int fieldID; protected String fieldName; protected ObjectInspector fieldObjectInspector; + protected String fieldComment; public MyField(int fieldID, String fieldName, ObjectInspector fieldObjectInspector) { @@ -54,6 +55,12 @@ public class StandardStructObjectInspector extends this.fieldObjectInspector = fieldObjectInspector; } + public MyField(int fieldID, String fieldName, + ObjectInspector fieldObjectInspector, String fieldComment) { + this(fieldID, fieldName, fieldObjectInspector); + this.fieldComment = fieldComment; + } + public int getFieldID() { return fieldID; } @@ -66,6 +73,10 @@ public class StandardStructObjectInspector extends return fieldObjectInspector; } + public String getFieldComment() { + return fieldComment; + } + @Override public String toString() { return "" + fieldID + ":" + fieldName; @@ -83,17 +94,30 @@ public class StandardStructObjectInspector extends */ protected StandardStructObjectInspector(List structFieldNames, List structFieldObjectInspectors) { - init(structFieldNames, structFieldObjectInspectors); + init(structFieldNames, structFieldObjectInspectors, null); + } + + /** + * Call ObjectInspectorFactory.getStandardListObjectInspector instead. + */ + protected StandardStructObjectInspector(List structFieldNames, + List structFieldObjectInspectors, + List structFieldComments) { + init(structFieldNames, structFieldObjectInspectors, structFieldComments); } protected void init(List structFieldNames, - List structFieldObjectInspectors) { + List structFieldObjectInspectors, + List structFieldComments) { assert (structFieldNames.size() == structFieldObjectInspectors.size()); + assert (structFieldComments == null || + (structFieldNames.size() == structFieldComments.size())); fields = new ArrayList(structFieldNames.size()); for (int i = 0; i < structFieldNames.size(); i++) { fields.add(new MyField(i, structFieldNames.get(i), - structFieldObjectInspectors.get(i))); + structFieldObjectInspectors.get(i), + structFieldComments == null ? null : structFieldComments.get(i))); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java index 62c3017..67827d6 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StructField.java @@ -36,4 +36,8 @@ public interface StructField { */ ObjectInspector getFieldObjectInspector(); + /** + * Get the comment for the field. May be null if no comment provided. + */ + String getFieldComment(); } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java index 76ff736..6e3dfea 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java @@ -55,6 +55,10 @@ public class UnionStructObjectInspector extends StructObjectInspector { public ObjectInspector getFieldObjectInspector() { return structField.getFieldObjectInspector(); } + + public String getFieldComment() { + return structField.getFieldComment(); + } } List unionObjectInspectors; diff --git serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java new file mode 100644 index 0000000..bb96a89 --- /dev/null +++ serde/src/test/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2; + +import junit.framework.TestCase; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +import java.util.ArrayList; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestSerdeWithFieldComments extends TestCase { + + private StructField mockedStructField(String name, String oiTypeName, + String comment) { + StructField m = mock(StructField.class); + when(m.getFieldName()).thenReturn(name); + + ObjectInspector oi = mock(ObjectInspector.class); + when(oi.getTypeName()).thenReturn(oiTypeName); + + when(m.getFieldObjectInspector()).thenReturn(oi); + when(m.getFieldComment()).thenReturn(comment); + + return m; + } + + public void testFieldComments() throws MetaException, SerDeException { + StructObjectInspector mockSOI = mock(StructObjectInspector.class); + when(mockSOI.getCategory()).thenReturn(ObjectInspector.Category.STRUCT); + List fieldRefs = new ArrayList(); + // Add field with a comment... + fieldRefs.add(mockedStructField("first", "type name 1", "this is a comment")); + // ... and one without + fieldRefs.add(mockedStructField("second", "type name 2", null)); + + when(mockSOI.getAllStructFieldRefs()).thenReturn(fieldRefs); + + Deserializer mockDe = mock(Deserializer.class); + when(mockDe.getObjectInspector()).thenReturn(mockSOI); + List result = + MetaStoreUtils.getFieldsFromDeserializer("testTable", mockDe); + + assertEquals(2, result.size()); + assertEquals("first", result.get(0).getName()); + assertEquals("this is a comment", result.get(0).getComment()); + assertEquals("second", result.get(1).getName()); + assertEquals("from deserializer", result.get(1).getComment()); + } +} diff --git serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java index f139ea5..cc0723d 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java +++ serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java @@ -263,75 +263,9 @@ public class TestStandardObjectInspectors extends TestCase { @SuppressWarnings("unchecked") public void testStandardStructObjectInspector() throws Throwable { try { - ArrayList fieldNames = new ArrayList(); - fieldNames.add("firstInteger"); - fieldNames.add("secondString"); - fieldNames.add("thirdBoolean"); - ArrayList fieldObjectInspectors = new ArrayList(); - fieldObjectInspectors - .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); - fieldObjectInspectors - .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); - fieldObjectInspectors - .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); - - StandardStructObjectInspector soi1 = ObjectInspectorFactory - .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors); - StandardStructObjectInspector soi2 = ObjectInspectorFactory - .getStandardStructObjectInspector((ArrayList) fieldNames - .clone(), (ArrayList) fieldObjectInspectors - .clone()); - assertEquals(soi1, soi2); - - // metadata - assertEquals(Category.STRUCT, soi1.getCategory()); - List fields = soi1.getAllStructFieldRefs(); - assertEquals(3, fields.size()); - for (int i = 0; i < 3; i++) { - assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i) - .getFieldName()); - assertEquals(fieldObjectInspectors.get(i), fields.get(i) - .getFieldObjectInspector()); - } - assertEquals(fields.get(1), soi1.getStructFieldRef("secondString")); - StringBuilder structTypeName = new StringBuilder(); - structTypeName.append("struct<"); - for (int i = 0; i < fields.size(); i++) { - if (i > 0) { - structTypeName.append(","); - } - structTypeName.append(fields.get(i).getFieldName()); - structTypeName.append(":"); - structTypeName.append(fields.get(i).getFieldObjectInspector() - .getTypeName()); - } - structTypeName.append(">"); - assertEquals(structTypeName.toString(), soi1.getTypeName()); - - // null - assertNull(soi1.getStructFieldData(null, fields.get(0))); - assertNull(soi1.getStructFieldData(null, fields.get(1))); - assertNull(soi1.getStructFieldData(null, fields.get(2))); - assertNull(soi1.getStructFieldsDataAsList(null)); - - // HashStruct - ArrayList struct = new ArrayList(3); - struct.add(1); - struct.add("two"); - struct.add(true); - - assertEquals(1, soi1.getStructFieldData(struct, fields.get(0))); - assertEquals("two", soi1.getStructFieldData(struct, fields.get(1))); - assertEquals(true, soi1.getStructFieldData(struct, fields.get(2))); - - // Settable - Object struct3 = soi1.create(); - System.out.println(struct3); - soi1.setStructFieldData(struct3, fields.get(0), 1); - soi1.setStructFieldData(struct3, fields.get(1), "two"); - soi1.setStructFieldData(struct3, fields.get(2), true); - assertEquals(struct, struct3); - + // Test StandardObjectInspector both with field comments and without + doStandardObjectInspectorTest(true); + doStandardObjectInspectorTest(false); } catch (Throwable e) { e.printStackTrace(); throw e; @@ -339,6 +273,95 @@ public class TestStandardObjectInspectors extends TestCase { } + private void doStandardObjectInspectorTest(boolean testComments) { + ArrayList fieldNames = new ArrayList(); + fieldNames.add("firstInteger"); + fieldNames.add("secondString"); + fieldNames.add("thirdBoolean"); + ArrayList fieldObjectInspectors = new ArrayList(); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + fieldObjectInspectors + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector); + ArrayList fieldComments = new ArrayList(3); + if(testComments) { + fieldComments.add("firstInteger comment"); + fieldComments.add("secondString comment"); + fieldComments.add("thirdBoolean comment"); + } else { // should have null for non-specified comments + for(int i = 0; i < 3; i++) fieldComments.add(null); + } + + StandardStructObjectInspector soi1 = testComments ? + ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors, + fieldComments) + : ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames, fieldObjectInspectors); + StandardStructObjectInspector soi2 = testComments ? + ObjectInspectorFactory + .getStandardStructObjectInspector((ArrayList) fieldNames + .clone(), (ArrayList) fieldObjectInspectors + .clone(), (ArrayList)fieldComments.clone()) + : ObjectInspectorFactory + .getStandardStructObjectInspector((ArrayList) fieldNames + .clone(), (ArrayList) fieldObjectInspectors + .clone()); + assertEquals(soi1, soi2); + + // metadata + assertEquals(Category.STRUCT, soi1.getCategory()); + List fields = soi1.getAllStructFieldRefs(); + assertEquals(3, fields.size()); + for (int i = 0; i < 3; i++) { + assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i) + .getFieldName()); + assertEquals(fieldObjectInspectors.get(i), fields.get(i) + .getFieldObjectInspector()); + assertEquals(fieldComments.get(i), fields.get(i).getFieldComment()); + } + assertEquals(fields.get(1), soi1.getStructFieldRef("secondString")); + StringBuilder structTypeName = new StringBuilder(); + structTypeName.append("struct<"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + structTypeName.append(","); + } + structTypeName.append(fields.get(i).getFieldName()); + structTypeName.append(":"); + structTypeName.append(fields.get(i).getFieldObjectInspector() + .getTypeName()); + } + structTypeName.append(">"); + assertEquals(structTypeName.toString(), soi1.getTypeName()); + + // null + assertNull(soi1.getStructFieldData(null, fields.get(0))); + assertNull(soi1.getStructFieldData(null, fields.get(1))); + assertNull(soi1.getStructFieldData(null, fields.get(2))); + assertNull(soi1.getStructFieldsDataAsList(null)); + + // HashStruct + ArrayList struct = new ArrayList(3); + struct.add(1); + struct.add("two"); + struct.add(true); + + assertEquals(1, soi1.getStructFieldData(struct, fields.get(0))); + assertEquals("two", soi1.getStructFieldData(struct, fields.get(1))); + assertEquals(true, soi1.getStructFieldData(struct, fields.get(2))); + + // Settable + Object struct3 = soi1.create(); + System.out.println(struct3); + soi1.setStructFieldData(struct3, fields.get(0), 1); + soi1.setStructFieldData(struct3, fields.get(1), "two"); + soi1.setStructFieldData(struct3, fields.get(2), true); + assertEquals(struct, struct3); + } + @SuppressWarnings("unchecked") public void testStandardUnionObjectInspector() throws Throwable { try {