commit b897c13a72b8e8dc7c5d9443276d2080447500ca Author: Vihang Karajgaonkar Date: Thu Feb 22 13:10:03 2018 -0800 Moved TypeInfo to standalone-metastore Introduce Avro storage schema reader Fixed a bug in typeInfoParser ColumnType is missing TypeStampWithLocalTimeZone diff --git a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java index 7ad6a45b7d7871d469e0f4272e75feb64c91e030..7e6d21b076226eecaf1b81c2dc8fefc8822d7550 100644 --- a/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java +++ b/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java @@ -77,7 +77,7 @@ public AccumuloRowSerializer(int primaryKeyOffset, LazySerDeParameters serDePara public Mutation serialize(Object obj, ObjectInspector objInspector) throws SerDeException, IOException { - if (objInspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); } @@ -207,8 +207,8 @@ protected void serializeColumnMapping(HiveAccumuloMapColumnMapping columnMapping String rowIdMappingType = rowIdMapping.getColumnType(); TypeInfo rowIdTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(rowIdMappingType); - if (!rowIdFieldOI.getCategory().equals(ObjectInspector.Category.PRIMITIVE) - && rowIdTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (!rowIdFieldOI.getCategory().equals(TypeInfo.Category.PRIMITIVE) + && rowIdTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { // we always serialize the String type using the escaped algorithm for LazyString writeString(output, SerDeUtils.getJSONString(rowId, rowIdFieldOI), PrimitiveObjectInspectorFactory.javaStringObjectInspector); @@ -242,7 +242,7 @@ protected void serializeColumnMapping(HiveAccumuloMapColumnMapping columnMapping output.reset(); // Start by only serializing primitives as-is - if (fieldObjectInspector.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (fieldObjectInspector.getCategory() == TypeInfo.Category.PRIMITIVE) { writeSerializedPrimitive((PrimitiveObjectInspector) fieldObjectInspector, output, value, mapping.getEncoding()); } else { diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java index f885eba7f10fc88396b1bccdada5e3fbb1eca7fb..dab5f7b93b4cd710589a364055c796803759bdae 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/DelimitedAccumuloRowIdFactory.java @@ -80,7 +80,7 @@ public LazyObjectBase createRowId(ObjectInspector inspector) throws SerDeExcepti public byte[] serializeRowId(Object object, StructField field, ByteStream.Output output) throws IOException { ObjectInspector inspector = field.getFieldObjectInspector(); - if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (inspector.getCategory() != TypeInfo.Category.STRUCT) { throw new IllegalStateException("invalid type value " + inspector.getTypeName()); } diff --git a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java index 85bc960237736192306ebe4239391a93ca9ac3de..f275f280beaf9b9defae94fad04f6aa39ed296e0 100644 --- a/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java +++ b/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/TestAccumuloRowSerializer.java @@ -78,7 +78,7 @@ public void testBufferResetBeforeUse() throws IOException { Mockito.any(ByteStream.Output.class), Mockito.any(ColumnMapping.class))) .thenCallRealMethod(); - Mockito.when(fieldObjectInspector.getCategory()).thenReturn(ObjectInspector.Category.PRIMITIVE); + Mockito.when(fieldObjectInspector.getCategory()).thenReturn(TypeInfo.Category.PRIMITIVE); Mockito.when(fieldObjectInspector.getPrimitiveCategory()).thenReturn(PrimitiveCategory.STRING); Mockito.when(fieldObjectInspector.getPrimitiveWritableObject(Mockito.any(Object.class))) .thenReturn(new Text(object)); diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFAdd10.java b/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFAdd10.java index 408ad63453f544f48f35088b1c1bf9ac83002d33..0d8a44a8bf9b1f416556a45a5ecd5f37c7003f99 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFAdd10.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFAdd10.java @@ -29,11 +29,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -63,7 +63,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "ADD10() requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "ADD10 only takes primitive types, got " + arguments[0].getTypeName()); } diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java b/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java index 85888d9bf77f05d326b34f12fb07f1636c0c31b3..67e14bb2a0fb55c84c5061c512fc0364efab9790 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/genericudf/example/GenericUDFDBOutput.java @@ -22,6 +22,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -86,7 +87,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) // this should be connection url,username,password,query,column1[,columnn]* for (int i = 0; i < 4; i++) { - if (arguments[i].getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (arguments[i].getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) arguments[i]); if (!(poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.STRING)) { @@ -98,7 +99,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) } } for (int i = 4; i < arguments.length; i++) { - if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[i].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The argument of function should be primative" + ", but \"" + arguments[i].getTypeName() + "\" is found"); diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java index bb6d7792db711cc345fb62101b871f9cbb377323..ccfdc642d92f27082e008d67d00541a53011b276 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/serde2/TypedBytesSerDe.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -125,7 +124,7 @@ public void initialize(Configuration conf, Properties tbl) // All columns have to be primitive. for (int c = 0; c < numColumns; c++) { - if (columnTypes.get(c).getCategory() != Category.PRIMITIVE) { + if (columnTypes.get(c).getCategory() != TypeInfo.Category.PRIMITIVE) { throw new SerDeException(getClass().getName() + " only accepts primitive columns, but column[" + c + "] named " + columnNames.get(c) + " has category " diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java index 70f28cbc5135fdcc692e137749fcc9b7085292d9..d267484bb7a0d4ed6982308d6f3c8a2c2113c786 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFExplode2.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * GenericUDTFExplode2. @@ -52,7 +53,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) throw new UDFArgumentException("explode() takes only one argument"); } - if (args[0].getCategory() != ObjectInspector.Category.LIST) { + if (args[0].getCategory() != TypeInfo.Category.LIST) { throw new UDFArgumentException("explode() takes an array as a parameter"); } listOI = (ListObjectInspector) args[0]; diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index 3899bffca58d99addda66a00cfc195d4f53828d7..6831df254f06dd1900893c086a4427841a339690 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.BooleanWritable; @@ -446,7 +447,7 @@ private void inferSchema(GroupByQuery query, TimestampLocalTZTypeInfo timeColumn @Override public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { - if (objectInspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (objectInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objectInspector.getTypeName()); diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java index f1887b549908a2328d5c8a32535097afa40e6f1e..f4bc09f5556ccdf1c44a617f3d302723388941a1 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/ColumnMappings.java @@ -33,13 +33,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import com.google.common.collect.Iterators; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; public class ColumnMappings implements Iterable { @@ -121,9 +121,9 @@ void setHiveColumnDescription(String serdeName, colMap.columnType = columnTypes.get(i); if (colMap.qualifierName == null && !colMap.hbaseRowKey && !colMap.hbaseTimestamp) { TypeInfo typeInfo = columnTypes.get(i); - if ((typeInfo.getCategory() != ObjectInspector.Category.MAP) || + if ((typeInfo.getCategory() != TypeInfo.Category.MAP) || (((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getCategory() - != ObjectInspector.Category.PRIMITIVE)) { + != TypeInfo.Category.PRIMITIVE)) { throw new SerDeException( serdeName + ": hbase column family '" + colMap.familyName @@ -192,24 +192,24 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx if (storageInfo == null) { // use the table default storage specification - if (colType.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (colType.getCategory() == TypeInfo.Category.PRIMITIVE) { if (!colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { colMap.binaryStorage.add(tableBinaryStorage); } else { colMap.binaryStorage.add(false); } - } else if (colType.getCategory() == ObjectInspector.Category.MAP) { + } else if (colType.getCategory() == TypeInfo.Category.MAP) { TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo(); TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo(); - if (keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && + if (keyTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { colMap.binaryStorage.add(tableBinaryStorage); } else { colMap.binaryStorage.add(false); } - if (valueTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && + if (valueTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { colMap.binaryStorage.add(tableBinaryStorage); } else { @@ -223,7 +223,7 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx // we have a storage specification for a primitive column type String storageOption = storageInfo[0]; - if ((colType.getCategory() == ObjectInspector.Category.MAP) || + if ((colType.getCategory() == TypeInfo.Category.MAP) || !(storageOption.equals("-") || "string".startsWith(storageOption) || "binary".startsWith(storageOption))) { throw new SerDeException("Error: A column storage specification is one of the following:" @@ -232,7 +232,7 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx + colMap.columnName); } - if (colType.getCategory() == ObjectInspector.Category.PRIMITIVE && + if (colType.getCategory() == TypeInfo.Category.PRIMITIVE && !colType.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { if ("-".equals(storageOption)) { @@ -252,7 +252,7 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx String keyStorage = storageInfo[0]; String valStorage = storageInfo[1]; - if ((colType.getCategory() != ObjectInspector.Category.MAP) || + if ((colType.getCategory() != TypeInfo.Category.MAP) || !(keyStorage.equals("-") || "string".startsWith(keyStorage) || "binary".startsWith(keyStorage)) || !(valStorage.equals("-") || "string".startsWith(valStorage) || @@ -270,7 +270,7 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx TypeInfo keyTypeInfo = ((MapTypeInfo) colType).getMapKeyTypeInfo(); TypeInfo valueTypeInfo = ((MapTypeInfo) colType).getMapValueTypeInfo(); - if (keyTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && + if (keyTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && !keyTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { if (keyStorage.equals("-")) { @@ -284,7 +284,7 @@ void parseColumnStorageTypes(String hbaseTableDefaultStorageType) throws SerDeEx colMap.binaryStorage.add(false); } - if (valueTypeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE && + if (valueTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && !valueTypeInfo.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { if (valStorage.equals("-")) { colMap.binaryStorage.add(tableBinaryStorage); @@ -404,12 +404,12 @@ public boolean isDoPrefixCut(){ return doPrefixCut; } - public boolean isCategory(ObjectInspector.Category category) { + public boolean isCategory(Category category) { return columnType.getCategory() == category; } public boolean isCategory(PrimitiveCategory category) { - return columnType.getCategory() == ObjectInspector.Category.PRIMITIVE && + return columnType.getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo)columnType).getPrimitiveCategory() == category; } diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java index be6aa1e2f877922c15f87f104d4321a57a03d044..74496e54f160fd9b99966a521d5359b53a3ab6bf 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Writable; public class HBaseRowSerializer { @@ -76,7 +77,7 @@ public HBaseRowSerializer(HBaseSerDeParameters hbaseParam) { } public Writable serialize(Object obj, ObjectInspector objInspector) throws Exception { - if (objInspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -128,8 +129,8 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws Excep } ObjectInspector keyFieldOI = keyField.getFieldObjectInspector(); - if (!keyFieldOI.getCategory().equals(ObjectInspector.Category.PRIMITIVE) && - keyMapping.isCategory(ObjectInspector.Category.PRIMITIVE)) { + if (!keyFieldOI.getCategory().equals(TypeInfo.Category.PRIMITIVE) && + keyMapping.isCategory(TypeInfo.Category.PRIMITIVE)) { // we always serialize the String type using the escaped algorithm for LazyString return serialize(SerDeUtils.getJSONString(keyValue, keyFieldOI), PrimitiveObjectInspectorFactory.javaStringObjectInspector, 1, false); @@ -183,8 +184,8 @@ private void serializeField( // the field is declared as a primitive in initialization, serialize // the data to JSON string. Otherwise serialize the data in the // delimited way. - if (!foi.getCategory().equals(ObjectInspector.Category.PRIMITIVE) - && colMap.isCategory(ObjectInspector.Category.PRIMITIVE)) { + if (!foi.getCategory().equals(TypeInfo.Category.PRIMITIVE) + && colMap.isCategory(TypeInfo.Category.PRIMITIVE)) { // we always serialize the String type using the escaped algorithm for LazyString bytes = serialize(SerDeUtils.getJSONString(value, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, 1, false); @@ -216,7 +217,7 @@ private void serializeField( private byte[] serialize(Object obj, ObjectInspector objInspector, int level, boolean writeBinary) throws IOException { output.reset(); - if (objInspector.getCategory() == ObjectInspector.Category.PRIMITIVE && writeBinary) { + if (objInspector.getCategory() == TypeInfo.Category.PRIMITIVE && writeBinary) { LazyUtils.writePrimitive(output, obj, (PrimitiveObjectInspector) objInspector); } else { if (!serialize(obj, objInspector, level, output)) { diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java index 8c9271c51e338a8ca53c9710a2f24eb24baf7ccd..909b3f3327f2087be9377bcbc9918c9fe5a3f19e 100644 --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer; import org.apache.hadoop.hive.ql.index.IndexSearchCondition; @@ -58,7 +57,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.AbstractSerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -425,7 +423,7 @@ public static DecomposedPredicate decomposePredicate( continue; } TypeInfo typeInfo = searchConditions.get(0).getColumnDesc().getTypeInfo(); - if (typeInfo.getCategory() == Category.PRIMITIVE && PrimitiveObjectInspectorUtils.getPrimitiveGrouping( + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) == PrimitiveGrouping.NUMERIC_GROUP) { // If the predicate is on a numeric column, and it specifies an // open range e.g. key < 20 , we do not support conversion, as negative diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java index 7136b466fc26ffc62fa02082a286fa0d7c562b5d..101fd2efaefec2c6bf3dbfa8cbda03afccc113a2 100644 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory.java @@ -55,7 +55,7 @@ public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException @Override public byte[] serializeKey(Object object, StructField field) throws IOException { ObjectInspector inspector = field.getFieldObjectInspector(); - if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (inspector.getCategory() != TypeInfo.Category.STRUCT) { throw new IllegalStateException("invalid type value " + inspector.getTypeName()); } output.reset(); diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java index 3a4d4734ae6f2f2aec45b66ced11480245da986a..9b2a07550b51260ea1e3d5447a843d68387028c2 100644 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory2.java @@ -63,7 +63,7 @@ public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException @Override public byte[] serializeKey(Object object, StructField field) throws IOException { ObjectInspector inspector = field.getFieldObjectInspector(); - if (inspector.getCategory() != ObjectInspector.Category.STRUCT) { + if (inspector.getCategory() != TypeInfo.Category.STRUCT) { throw new IllegalStateException("invalid type value " + inspector.getTypeName()); } output.reset(); diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java index 7ac0be6fb6edf2895864a2ecdbc2ef7a336e970f..dfaeff0bc284d1ccb31330b9cf9c10d0593f58bc 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordSerDe.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -146,7 +145,7 @@ public Object deserialize(Writable data) throws SerDeException { @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -195,13 +194,13 @@ public static Object serializeField(Object field, ObjectInspector fieldObjectIns throws SerDeException { Object res; - if (fieldObjectInspector.getCategory() == Category.PRIMITIVE) { + if (fieldObjectInspector.getCategory() == TypeInfo.Category.PRIMITIVE) { res = serializePrimitiveField(field, fieldObjectInspector); - } else if (fieldObjectInspector.getCategory() == Category.STRUCT) { + } else if (fieldObjectInspector.getCategory() == TypeInfo.Category.STRUCT) { res = serializeStruct(field, (StructObjectInspector) fieldObjectInspector); - } else if (fieldObjectInspector.getCategory() == Category.LIST) { + } else if (fieldObjectInspector.getCategory() == TypeInfo.Category.LIST) { res = serializeList(field, (ListObjectInspector) fieldObjectInspector); - } else if (fieldObjectInspector.getCategory() == Category.MAP) { + } else if (fieldObjectInspector.getCategory() == TypeInfo.Category.MAP) { res = serializeMap(field, (MapObjectInspector) fieldObjectInspector); } else { throw new SerDeException(HCatRecordSerDe.class.toString() @@ -239,25 +238,25 @@ public static Object serializeField(Object field, ObjectInspector fieldObjectIns } ObjectInspector eloi = loi.getListElementObjectInspector(); - if (eloi.getCategory() == Category.PRIMITIVE) { + if (eloi.getCategory() == TypeInfo.Category.PRIMITIVE) { List list = new ArrayList(l.size()); for (int i = 0; i < l.size(); i++) { list.add(((PrimitiveObjectInspector) eloi).getPrimitiveJavaObject(l.get(i))); } return list; - } else if (eloi.getCategory() == Category.STRUCT) { + } else if (eloi.getCategory() == TypeInfo.Category.STRUCT) { List> list = new ArrayList>(l.size()); for (int i = 0; i < l.size(); i++) { list.add(serializeStruct(l.get(i), (StructObjectInspector) eloi)); } return list; - } else if (eloi.getCategory() == Category.LIST) { + } else if (eloi.getCategory() == TypeInfo.Category.LIST) { List> list = new ArrayList>(l.size()); for (int i = 0; i < l.size(); i++) { list.add(serializeList(l.get(i), (ListObjectInspector) eloi)); } return list; - } else if (eloi.getCategory() == Category.MAP) { + } else if (eloi.getCategory() == TypeInfo.Category.MAP) { List> list = new ArrayList>(l.size()); for (int i = 0; i < l.size(); i++) { list.add(serializeMap(l.get(i), (MapObjectInspector) eloi)); diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/LazyHCatRecord.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/LazyHCatRecord.java index 052d47e6838fa5aa4d4abc8dd6265e844ae7ed85..3746c8540641abe63fb0ce288a3d569ec8529cd2 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/LazyHCatRecord.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/LazyHCatRecord.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatSchema; import org.slf4j.Logger; @@ -113,7 +113,7 @@ public void copy(HCatRecord r) throws HCatException { } public LazyHCatRecord(Object wrappedObject, ObjectInspector oi) throws Exception { - if (oi.getCategory() != Category.STRUCT) { + if (oi.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only make a lazy hcat record from " + "objects of struct types, but we got: " + oi.getTypeName()); diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java index 350221528e857c6c580ba028e34225928b6d0c39..0c9ed438ba45927e4fc7f0378988ffade3df6b0e 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatFieldSchema.java @@ -23,11 +23,10 @@ import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hive.hcatalog.common.HCatException; -import org.apache.hive.hcatalog.common.HCatUtil; @InterfaceAudience.Public @InterfaceStability.Stable @@ -57,24 +56,24 @@ DATE(PrimitiveObjectInspector.PrimitiveCategory.DATE), TIMESTAMP(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP), - ARRAY(ObjectInspector.Category.LIST), - MAP(ObjectInspector.Category.MAP), - STRUCT(ObjectInspector.Category.STRUCT); + ARRAY(TypeInfo.Category.LIST), + MAP(TypeInfo.Category.MAP), + STRUCT(TypeInfo.Category.STRUCT); - private final ObjectInspector.Category category; + private final TypeInfo.Category category; private final PrimitiveObjectInspector.PrimitiveCategory primitiveCategory; - private Type(ObjectInspector.Category cat) { + private Type(TypeInfo.Category cat) { category = cat; primitiveCategory = null; - assert category != ObjectInspector.Category.PRIMITIVE : + assert category != TypeInfo.Category.PRIMITIVE : "This c'tor should be used for complex category types"; } private Type(PrimitiveObjectInspector.PrimitiveCategory primCat) { - category = ObjectInspector.Category.PRIMITIVE; + category = TypeInfo.Category.PRIMITIVE; primitiveCategory = primCat; } - public ObjectInspector.Category getCategory() { + public TypeInfo.Category getCategory() { return category; } /** @@ -95,7 +94,7 @@ public static Type getPrimitiveHType(PrimitiveTypeInfo basePrimitiveTypeInfo) { public static int numPrimitiveTypes() { int numPrimitives = 0; for(Type t : values()) { - if(t.category == ObjectInspector.Category.PRIMITIVE) { + if(t.category == TypeInfo.Category.PRIMITIVE) { numPrimitives++; } } diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java index 999abcbfed03851bdab760b29069ccd5f6120227..4570f213055d6db72bf1613047082c5675228117 100644 --- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java +++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/schema/HCatSchemaUtils.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Schema; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -118,15 +118,15 @@ public static HCatFieldSchema getHCatFieldSchema(FieldSchema fs) throws HCatExce private static HCatFieldSchema getHCatFieldSchema(String fieldName, TypeInfo fieldTypeInfo, String comment) throws HCatException { Category typeCategory = fieldTypeInfo.getCategory(); HCatFieldSchema hCatFieldSchema; - if (Category.PRIMITIVE == typeCategory) { + if (TypeInfo.Category.PRIMITIVE == typeCategory) { hCatFieldSchema = new HCatFieldSchema(fieldName, (PrimitiveTypeInfo)fieldTypeInfo, comment); - } else if (Category.STRUCT == typeCategory) { + } else if (TypeInfo.Category.STRUCT == typeCategory) { HCatSchema subSchema = constructHCatSchema((StructTypeInfo) fieldTypeInfo); hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.STRUCT, subSchema, comment); - } else if (Category.LIST == typeCategory) { + } else if (TypeInfo.Category.LIST == typeCategory) { HCatSchema subSchema = getHCatSchema(((ListTypeInfo) fieldTypeInfo).getListElementTypeInfo()); hCatFieldSchema = new HCatFieldSchema(fieldName, HCatFieldSchema.Type.ARRAY, subSchema, comment); - } else if (Category.MAP == typeCategory) { + } else if (TypeInfo.Category.MAP == typeCategory) { HCatSchema subSchema = getHCatSchema(((MapTypeInfo) fieldTypeInfo).getMapValueTypeInfo()); hCatFieldSchema = HCatFieldSchema.createMapTypeFieldSchema(fieldName, (PrimitiveTypeInfo)((MapTypeInfo)fieldTypeInfo).getMapKeyTypeInfo(), subSchema, comment); @@ -159,16 +159,16 @@ private static HCatSchema constructHCatSchema(StructTypeInfo stypeInfo) throws H public static HCatSchema getHCatSchema(TypeInfo typeInfo) throws HCatException { Category typeCategory = typeInfo.getCategory(); HCatSchema hCatSchema; - if (Category.PRIMITIVE == typeCategory) { + if (TypeInfo.Category.PRIMITIVE == typeCategory) { hCatSchema = getStructSchemaBuilder().addField(new HCatFieldSchema(null, (PrimitiveTypeInfo)typeInfo, null)).build(); - } else if (Category.STRUCT == typeCategory) { + } else if (TypeInfo.Category.STRUCT == typeCategory) { HCatSchema subSchema = constructHCatSchema((StructTypeInfo) typeInfo); hCatSchema = getStructSchemaBuilder().addField(new HCatFieldSchema(null, Type.STRUCT, subSchema, null)).build(); - } else if (Category.LIST == typeCategory) { + } else if (TypeInfo.Category.LIST == typeCategory) { CollectionBuilder builder = getListSchemaBuilder(); builder.addField(getHCatFieldSchema(null, ((ListTypeInfo) typeInfo).getListElementTypeInfo(), null)); hCatSchema = new HCatSchema(Arrays.asList(new HCatFieldSchema("", Type.ARRAY, builder.build(), ""))); - } else if (Category.MAP == typeCategory) { + } else if (TypeInfo.Category.MAP == typeCategory) { HCatSchema subSchema = getHCatSchema(((MapTypeInfo) typeInfo).getMapValueTypeInfo()); MapBuilder builder = getMapSchemaBuilder(); hCatSchema = builder.withKeyType((PrimitiveTypeInfo)((MapTypeInfo) typeInfo).getMapKeyTypeInfo()) diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableListObjectInspector1.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableListObjectInspector1.java index 695d3ffb51fb433cd29282288ad450741c12296e..58be243f6aceafc1040ed2329d00196b1a9c4b2b 100644 --- a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableListObjectInspector1.java +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableListObjectInspector1.java @@ -22,6 +22,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; public class CustomNonSettableListObjectInspector1 implements ListObjectInspector { @@ -36,7 +38,7 @@ protected CustomNonSettableListObjectInspector1( } public final Category getCategory() { - return Category.LIST; + return TypeInfo.Category.LIST; } // without data diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableStructObjectInspector1.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableStructObjectInspector1.java index 84caf5e401b64629324871ef53cd8dbde931451c..a89d6092ee31f50ccf735eff531614ccb6f4af24 100644 --- a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableStructObjectInspector1.java +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableStructObjectInspector1.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -107,7 +109,7 @@ public String getTypeName() { } public final Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } // Without Data diff --git a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java index 61179ba2946128d88d1eddc036c07198ce3f0c9c..64d1124647cb3caed886c3be7ce0a007b0a02062 100644 --- a/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java +++ b/itests/custom-serde/src/main/java/org/apache/hadoop/hive/serde2/CustomNonSettableUnionObjectInspector1.java @@ -23,6 +23,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; public class CustomNonSettableUnionObjectInspector1 implements UnionObjectInspector{ @@ -81,7 +83,7 @@ public Object getField(Object o) { } public Category getCategory() { - return Category.UNION; + return TypeInfo.Category.UNION; } public String getTypeName() { diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java index b39871205d32586878fbecb7a2ec3263afe24c4d..ec4d14617431916ef09d5ca7c91221f7a1b6f477 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import java.util.ArrayList; import java.util.List; @@ -49,7 +50,7 @@ private StructField mockedStructField(String name, String oiTypeName, public void testFieldComments() throws MetaException, SerDeException { StructObjectInspector mockSOI = mock(StructObjectInspector.class); - when(mockSOI.getCategory()).thenReturn(ObjectInspector.Category.STRUCT); + when(mockSOI.getCategory()).thenReturn(TypeInfo.Category.STRUCT); List fieldRefs = new ArrayList(); // Add field with a comment... fieldRefs.add(mockedStructField("first", "type name 1", "this is a comment")); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java index 68a8e21307dba01e829868b5ccfd7a496d192780..b72be215b0c43e2b0f3e6d2c1c25b3719e976180 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniLlap.java @@ -27,7 +27,6 @@ import static org.junit.Assert.fail; import java.io.File; -import java.lang.reflect.Field; import java.math.BigDecimal; import java.net.URL; import java.sql.Connection; @@ -38,29 +37,12 @@ import java.sql.Statement; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; import java.util.UUID; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; @@ -68,30 +50,17 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.LlapRowRecordReader; import org.apache.hadoop.hive.llap.Row; import org.apache.hadoop.hive.llap.Schema; import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; import org.apache.hive.jdbc.miniHS2.MiniHS2; import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType; import org.apache.hadoop.hive.llap.LlapBaseInputFormat; import org.apache.hadoop.hive.llap.LlapRowInputFormat; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; - -import org.datanucleus.ClassLoaderResolver; -import org.datanucleus.NucleusContext; -import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; -import org.datanucleus.AbstractNucleusContext; + import org.junit.After; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; diff --git a/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java b/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java index d23e30d1457aa0a21a4de486af2bea26f9915953..931ddaa726411215eae7496b5bc143c23dfa23a9 100644 --- a/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java +++ b/itests/test-serde/src/main/java/org/apache/hadoop/hive/serde2/TestSerDe.java @@ -24,13 +24,13 @@ import java.util.List; import java.util.Properties; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.MetadataListStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -174,7 +174,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -188,7 +188,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe sb.append(separator); } Object column = soi.getStructFieldData(obj, fields.get(i)); - if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { + if (fields.get(i).getFieldObjectInspector().getCategory() == TypeInfo.Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumList.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumList.java index 5ec39ca5de09fee5b06088fb47cbf02a4827b1f0..12284e11fd3c0ac6cc536c061549ebd600fc2404 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumList.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumList.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.udf.generic; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -50,14 +51,14 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) "Exactly one argument is expected."); } - if (inspectors[0].getCategory() != ObjectInspector.Category.LIST) { + if (inspectors[0].getCategory() != TypeInfo.Category.LIST) { throw new UDFArgumentTypeException(0, "Argument should be a list type"); } ListObjectInspector listOI = (ListObjectInspector) inspectors[0]; ObjectInspector elementOI = listOI.getListElementObjectInspector(); - if (elementOI.getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (elementOI.getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + elementOI.getTypeName() + " is passed."); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java index b8f755f8956c89f13025afb3eff3561645bde84e..819c6978bcb4b0644071599735fc0e9c0e1aa199 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEvaluateNPE.java @@ -27,9 +27,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; /** diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java index 1cfbf3a86ec60e6612edd71cc6793f0526fbe1bd..7b5f095a21772ff62d1899a3fda1d98ab7f103b1 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapRowRecordReader.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -31,30 +30,19 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.RecordReader; -import org.apache.hadoop.mapred.Reporter; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.llap.Row; -import org.apache.hadoop.hive.llap.FieldDesc; -import org.apache.hadoop.hive.llap.Schema; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.io.HiveCharWritable; -import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; -import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -161,7 +149,7 @@ static Object convertValue(Object val, ObjectInspector oi) { } Object convertedVal = null; - ObjectInspector.Category oiCategory = oi.getCategory(); + Category oiCategory = oi.getCategory(); switch (oiCategory) { case PRIMITIVE: convertedVal = convertPrimitive(val, (PrimitiveObjectInspector) oi); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index d252279be973201227da52d8aecf83b3fcc4656b..042ffe3fcfef43294f3a265c0065b153ee351756 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.NullWritable; @@ -219,7 +218,7 @@ private static int determineQueueLimit( double totalWeight = 0; for (TypeInfo ti : typeInfos) { int colWeight = 1; - if (ti.getCategory() != Category.PRIMITIVE) { + if (ti.getCategory() != TypeInfo.Category.PRIMITIVE) { colWeight = COL_WEIGHT_COMPLEX; } else { PrimitiveTypeInfo pti = (PrimitiveTypeInfo)ti; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java index de19b1d78b4072e62dc3ebde2f3a79278f6ab129..5a90dcdbaf864a829f94f430624b4c14a2e8a6ed 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java @@ -53,7 +53,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.InputFormat; @@ -106,7 +107,7 @@ public static EncodingWriter create(InputFormat sourceIf, Deserializer ser } for (StructField sf : sourceOi.getAllStructFieldRefs()) { Category c = sf.getFieldObjectInspector().getCategory(); - if (c != Category.PRIMITIVE) { + if (c != TypeInfo.Category.PRIMITIVE) { LlapIoImpl.LOG.info("Not using VertorDeserializeOrcWriter: " + c + " is not supported"); return new DeserializerOrcWriter(serDe, sourceOi, allocSize); } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java index a66c13507abef42977dfdb315ff7d69404f67ac3..6c4dc85bff49afb5dc69d1ebdc6c9f6b85e9aed9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -178,7 +177,7 @@ static public Deserializer getDeserializer(Configuration conf, ArrayList str_fields = new ArrayList<>(); // rules on how to recurse the ObjectInspector based on its type - if (oi.getCategory() != Category.STRUCT) { + if (oi.getCategory() != TypeInfo.Category.STRUCT) { str_fields.add(new FieldSchema(last_name, oi.getTypeName(), FROM_SERIALIZER)); } else { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java index 59bcd5ca34d5083d357d7157abf3682399060a1a..05e10784a4ac075a458db30976d047447b12d7a0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java @@ -27,6 +27,10 @@ import java.util.List; +/** + * In order to use this Storage schema reader you should add the hive-serde jar in the classpath + * of the metastore. + */ public class SerDeStorageSchemaReader implements StorageSchemaReader { @Override public List readSchema(Table tbl, EnvironmentContext envContext, Configuration conf) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index 98bb938c1373dce3b0600011a823e6bee952cd4e..c104b11e6d73f89937d57411b8e65c47a48adf91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.Serializable; -import java.io.StringWriter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -78,31 +77,18 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyShim; import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyValue; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; - -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_TEMPORARY_TABLE_STORAGE; - /** * File Sink operator implementation. **/ @@ -880,7 +866,7 @@ public void process(Object row, int tag) throws HiveException { updateProgress(); // if DP is enabled, get the final output writers and prepare the real output row - assert inputObjInspectors[0].getCategory() == ObjectInspector.Category.STRUCT + assert inputObjInspectors[0].getCategory() == TypeInfo.Category.STRUCT : "input object inspector is not struct"; if (bDynParts) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index f7801bb5a43e25a4d841746bf129a397a8720703..b7bc80f6b6c4e71ee940fa0af44d59349b94ab12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -134,7 +134,6 @@ import org.apache.hadoop.hive.ql.udf.xml.UDFXPathString; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; @@ -741,7 +740,7 @@ public static TypeInfo getCommonClassForUnionAll(TypeInfo a, TypeInfo b) { if (a.equals(b)) { return a; } - if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + if (a.getCategory() != TypeInfo.Category.PRIMITIVE || b.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory(); @@ -794,7 +793,7 @@ public static synchronized TypeInfo getCommonClassForComparison(TypeInfo a, Type if (a.equals(b)) { return a; } - if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + if (a.getCategory() != TypeInfo.Category.PRIMITIVE || b.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory(); @@ -849,7 +848,7 @@ public static synchronized TypeInfo getCommonClassForComparison(TypeInfo a, Type } public static PrimitiveCategory getPrimitiveCommonCategory(TypeInfo a, TypeInfo b) { - if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) { + if (a.getCategory() != TypeInfo.Category.PRIMITIVE || b.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } @@ -928,7 +927,7 @@ public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) { return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, commonCat); } // It is not primitive; check if it is a struct and we can infer a common class - if (a.getCategory() == Category.STRUCT && b.getCategory() == Category.STRUCT) { + if (a.getCategory() == TypeInfo.Category.STRUCT && b.getCategory() == TypeInfo.Category.STRUCT) { return getCommonClassForStruct((StructTypeInfo)a, (StructTypeInfo)b); } return null; @@ -1109,8 +1108,8 @@ public static int matchCost(TypeInfo argumentPassed, // passing null matches everything return 0; } - if (argumentPassed.getCategory().equals(Category.LIST) - && argumentAccepted.getCategory().equals(Category.LIST)) { + if (argumentPassed.getCategory().equals(TypeInfo.Category.LIST) + && argumentAccepted.getCategory().equals(TypeInfo.Category.LIST)) { // lists are compatible if and only-if the elements are compatible TypeInfo argumentPassedElement = ((ListTypeInfo) argumentPassed) .getListElementTypeInfo(); @@ -1118,8 +1117,8 @@ public static int matchCost(TypeInfo argumentPassed, .getListElementTypeInfo(); return matchCost(argumentPassedElement, argumentAcceptedElement, exact); } - if (argumentPassed.getCategory().equals(Category.MAP) - && argumentAccepted.getCategory().equals(Category.MAP)) { + if (argumentPassed.getCategory().equals(TypeInfo.Category.MAP) + && argumentAccepted.getCategory().equals(TypeInfo.Category.MAP)) { // lists are compatible if and only-if the elements are compatible TypeInfo argumentPassedKey = ((MapTypeInfo) argumentPassed) .getMapKeyTypeInfo(); @@ -1173,8 +1172,8 @@ static void filterMethodsByTypeAffinity(List udfMethods, List // Check the affinity of the argument passed in with the accepted argument, // based on the PrimitiveGrouping TypeInfo passedType = argsPassedIter.next(); - if (acceptedType.getCategory() == Category.PRIMITIVE - && passedType.getCategory() == Category.PRIMITIVE) { + if (acceptedType.getCategory() == TypeInfo.Category.PRIMITIVE + && passedType.getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveGrouping acceptedPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveTypeInfo) acceptedType).getPrimitiveCategory()); PrimitiveGrouping passedPg = PrimitiveObjectInspectorUtils.getPrimitiveGrouping( @@ -1306,7 +1305,7 @@ public static Method getMethodInternal(Class udfClass, List mlist, bo boolean acceptedIsPrimitive = false; PrimitiveCategory acceptedPrimCat = PrimitiveCategory.UNKNOWN; - if (accepted.getCategory() == Category.PRIMITIVE) { + if (accepted.getCategory() == TypeInfo.Category.PRIMITIVE) { acceptedIsPrimitive = true; acceptedPrimCat = ((PrimitiveTypeInfo) accepted).getPrimitiveCategory(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java index 4b766382efcd2af104767ba98b1a1633898be9d1..352863a5b3227cc9be439e17772316fd2a547541 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java @@ -260,7 +260,7 @@ protected void initializeOp(Configuration hconf) throws HiveException { // the last field is the union field, if any StructField sf = keysfs.get(keysfs.size() - 1); if (sf.getFieldObjectInspector().getCategory().equals( - ObjectInspector.Category.UNION)) { + TypeInfo.Category.UNION)) { unionExprEval = ExprNodeEvaluatorFactory.get( new ExprNodeColumnDesc(TypeInfoUtils.getTypeInfoFromObjectInspector( sf.getFieldObjectInspector()), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java index 07991811f92bc7accae2fde23244f424bdd64c6b..40c1774a38b0256c64cd606ea6fb7338d6200e64 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java @@ -42,11 +42,11 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; import org.apache.hadoop.hive.ql.stats.StatsPublisher; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapred.JobConf; @@ -181,7 +181,7 @@ private void gatherStats(Object row) { List writable; List values; int dpStartCol; // the first position of partition column - assert inputObjInspectors[0].getCategory() == ObjectInspector.Category.STRUCT : "input object inspector is not struct"; + assert inputObjInspectors[0].getCategory() == TypeInfo.Category.STRUCT : "input object inspector is not struct"; writable = new ArrayList(conf.getPartColumns().size()); values = new ArrayList(conf.getPartColumns().size()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java index 033bbdb6a5b9b4b69c0b8ebcc1e00501449e11f3..55972dd624bf4f928b1ff3c75d8071a140a44315 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java @@ -50,7 +50,7 @@ import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -472,7 +472,7 @@ public static boolean hasComplexObjects(LazyBinaryStructObjectInspector lazyBina List fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (StructField field : fields) { - if (field.getFieldObjectInspector().getCategory() != Category.PRIMITIVE) { + if (field.getFieldObjectInspector().getCategory() != TypeInfo.Category.PRIMITIVE) { return true; } } @@ -492,7 +492,7 @@ public static boolean hasComplexObjects(LazyBinaryStructObjectInspector lazyBina ObjectInspector objectInspector = field.getFieldObjectInspector(); Category category = objectInspector.getCategory(); Object object = lazyBinaryStruct.getField(i); - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { objectArrayBuffer.set(i, object); } else { objectArrayBuffer.set(i, ObjectInspectorUtils.copyToStandardObject( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java index 6504a5f7b0a3f3d74c1074aac8b2f5aed3cd81fa..3ae9c12c59ff2918430f920a08543a6ce5a325f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -87,14 +86,14 @@ public static MapJoinKey read(Output output, MapJoinObjectSerDeContext context, } public static boolean isSupportedField(ObjectInspector foi) { - if (foi.getCategory() != Category.PRIMITIVE) return false; // not supported + if (foi.getCategory() != TypeInfo.Category.PRIMITIVE) return false; // not supported PrimitiveCategory pc = ((PrimitiveObjectInspector)foi).getPrimitiveCategory(); if (!SUPPORTED_PRIMITIVES.contains(pc)) return false; // not supported return true; } public static boolean isSupportedField(TypeInfo typeInfo) { - if (typeInfo.getCategory() != Category.PRIMITIVE) return false; // not supported + if (typeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) return false; // not supported PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; PrimitiveCategory pc = primitiveTypeInfo.getPrimitiveCategory(); if (!SUPPORTED_PRIMITIVES.contains(pc)) return false; // not supported diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java index 151d1b39eb17bba336bb02ab06f216361e4f6ff2..b92a6852d4b40e01affb02f9900be91dcfc68d8b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.llap.LlapDaemonInfo; import org.apache.hadoop.hive.ql.exec.MemoryMonitorInfo; import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -45,7 +46,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -316,7 +316,7 @@ private String describeOi(String desc, ObjectInspector keyOi) { for (StructField field : ((StructObjectInspector)keyOi).getAllStructFieldRefs()) { ObjectInspector oi = field.getFieldObjectInspector(); String cat = oi.getCategory().toString(); - if (oi.getCategory() == Category.PRIMITIVE) { + if (oi.getCategory() == TypeInfo.Category.PRIMITIVE) { cat = ((PrimitiveObjectInspector)oi).getPrimitiveCategory().toString(); } desc += field.getFieldName() + ":" + cat + ", "; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java index e96619cf86c9a802994513ee5bccecef6f58b0c6..0803a9fd6cc27b92cf1147943df4892174069fcb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java @@ -51,7 +51,7 @@ import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -141,7 +141,7 @@ private void initTargetEntry(int logicalColumnIndex, int projectionColumnNum, Ty isConvert[logicalColumnIndex] = false; projectionColumnNums[logicalColumnIndex] = projectionColumnNum; targetTypeInfos[logicalColumnIndex] = typeInfo; - if (typeInfo.getCategory() == Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { final PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; final PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory(); switch (primitiveCategory) { @@ -168,7 +168,7 @@ private void initConvertSourceEntry(int logicalColumnIndex, TypeInfo convertSour convertSourceOI[logicalColumnIndex] = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(convertSourceTypeInfo); - if (convertSourceCategory == Category.PRIMITIVE) { + if (convertSourceCategory == TypeInfo.Category.PRIMITIVE) { // These need to be based on the target. final PrimitiveCategory targetPrimitiveCategory = ((PrimitiveTypeInfo) targetTypeInfos[logicalColumnIndex]).getPrimitiveCategory(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index 8ea625e46605b73b57281c83c839de40ef8c0031..2c1af1ce7da345c6aaca46113eb358fa6e0782c4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; @@ -140,7 +140,7 @@ private VectorDeserializeRow() { public Field(PrimitiveCategory primitiveCategory, DataTypePhysicalVariation dataTypePhysicalVariation, int maxLength) { isPrimitive = true; - this.category = Category.PRIMITIVE; + this.category = TypeInfo.Category.PRIMITIVE; this.primitiveCategory = primitiveCategory; this.dataTypePhysicalVariation = dataTypePhysicalVariation; this.maxLength = maxLength; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java index 681d9cad7be95de1e332e3ee30b4343c3c3ec865..9504d7544d0dc937aad0e031aeee8139bc282e0e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -46,7 +45,7 @@ import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java index cb2efb7dbe270539bd7e398d4a42a80042d48b3c..ddb6ad7947732e67962d03b9e798d0fc2e203512 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.fast.SerializeWrite; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -109,7 +109,7 @@ private Field createField(TypeInfo typeInfo) { final Category category = typeInfo.getCategory(); field.category = category; field.typeInfo = typeInfo; - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { field.isPrimitive = true; field.primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); } else { @@ -153,7 +153,7 @@ public void init(List typeNames, int[] columnMap) throws HiveException { root = new Field(); root.isPrimitive = false; - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = count; root.objectInspector = null; @@ -176,7 +176,7 @@ public void init(List typeNames) throws HiveException { root = new Field(); root.isPrimitive = false; - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = count; root.objectInspector = null; @@ -194,7 +194,7 @@ public void init(TypeInfo[] typeInfos) root = new Field(); root.isPrimitive = false; - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = count; root.objectInspector = null; @@ -212,7 +212,7 @@ public void init(TypeInfo[] typeInfos, int[] columnMap) root = new Field(); root.isPrimitive = false; - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = count; root.objectInspector = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index 8264e8ad285deac29424bd1cb0bf626436d47c75..eba611dafb79f6ed3dea22603053ffb17b701963 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -73,7 +73,7 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; @@ -634,7 +634,7 @@ private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc case FILTER: // Evaluate the column as a boolean, converting if necessary. TypeInfo typeInfo = exprDesc.getTypeInfo(); - if (typeInfo.getCategory() == Category.PRIMITIVE && + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.BOOLEAN) { expr = new SelectColumnIsTrue(columnNum); } else { @@ -882,11 +882,11 @@ private TypeInfo getCommonTypeForChildExpressions(GenericUDF genericUdf, } } else if (genericUdf instanceof GenericUDFIn) { TypeInfo colTi = children.get(0).getTypeInfo(); - if (colTi.getCategory() != Category.PRIMITIVE) { + if (colTi.getCategory() != TypeInfo.Category.PRIMITIVE) { return colTi; // Handled later, only struct will be supported. } TypeInfo opTi = GenericUDFUtils.deriveInType(children); - if (opTi == null || opTi.getCategory() != Category.PRIMITIVE) { + if (opTi == null || opTi.getCategory() != TypeInfo.Category.PRIMITIVE) { throw new HiveException("Cannot vectorize IN() - common type is " + opTi); } if (((PrimitiveTypeInfo)colTi).getPrimitiveCategory() != @@ -2162,7 +2162,7 @@ private VectorExpression getStructInExpression(List childExpr, Exp for (int f = 0; f < fieldCount; f++) { TypeInfo fieldTypeInfo = fieldTypeInfos.get(f); // Only primitive fields supports for now. - if (fieldTypeInfo.getCategory() != Category.PRIMITIVE) { + if (fieldTypeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } @@ -2294,9 +2294,9 @@ private VectorExpression getInExpression(List childExpr, colType = VectorizationContext.mapTypeNameSynonyms(colType); TypeInfo colTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(colType); Category category = colTypeInfo.getCategory(); - if (category == Category.STRUCT) { + if (category == TypeInfo.Category.STRUCT) { return getStructInExpression(childExpr, colExpr, colTypeInfo, inChildren, mode, returnType); - } else if (category != Category.PRIMITIVE) { + } else if (category != TypeInfo.Category.PRIMITIVE) { return null; } @@ -3010,7 +3010,7 @@ private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc expr, Ve // Is output type a BOOLEAN? TypeInfo resultTypeInfo = expr.getTypeInfo(); - if (resultTypeInfo.getCategory() == Category.PRIMITIVE && + if (resultTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo) resultTypeInfo).getPrimitiveCategory() == PrimitiveCategory.BOOLEAN) { isFilter = true; } else { @@ -3285,13 +3285,13 @@ private Timestamp evaluateCastToTimestamp(ExprNodeDesc expr) throws HiveExceptio static String getScratchName(TypeInfo typeInfo) throws HiveException { // For now, leave DECIMAL precision/scale in the name so DecimalColumnVector scratch columns // don't need their precision/scale adjusted... - if (typeInfo.getCategory() == Category.PRIMITIVE && + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.DECIMAL) { return typeInfo.getTypeName(); } // And, for Complex Types, also leave the children types in place... - if (typeInfo.getCategory() != Category.PRIMITIVE) { + if (typeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { return typeInfo.getTypeName(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java index 66de847ef6759f0e85c719bb271c933f8074664f..573bd02c99eae56737903a8263a8ef2fca866503 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java @@ -23,22 +23,17 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; import org.apache.hadoop.hive.common.type.HiveChar; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DateWritable; @@ -50,18 +45,13 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -70,7 +60,6 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataOutputBuffer; @@ -79,7 +68,6 @@ import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; -import org.apache.hive.common.util.DateUtils; public class VectorizedBatchUtil { private static final Logger LOG = LoggerFactory.getLogger(VectorizedBatchUtil.class); @@ -322,7 +310,7 @@ private static void setVector(Object row, ObjectInspector foi = field.getFieldObjectInspector(); // Vectorization only supports PRIMITIVE data types. Assert the same - assert (foi.getCategory() == Category.PRIMITIVE); + assert (foi.getCategory() == TypeInfo.Category.PRIMITIVE); // Get writable object PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java index 92ec1ee8ba03c691ba00cd1e9a524a48b7dcab2b..9eacf0203de17d12066237a5a8ee575228739bb1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.HybridHashTableContainer.HashPartition; -import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer; import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorDeserializeRow; @@ -50,11 +49,8 @@ import org.apache.hadoop.hive.serde2.WriteBuffers.ByteSegmentRef; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.ByteStream.Output; /** @@ -447,7 +443,7 @@ private void setupSpillSerDe(VectorizedRowBatch batch) throws HiveException { for (int i = 0; i < projectionSize; i++) { int projectedColumn = projectedColumns.get(i); if (batch.cols[projectedColumn] != null && - inputObjInspectorsTypeInfos[i].getCategory() == Category.PRIMITIVE) { + inputObjInspectorsTypeInfos[i].getCategory() == TypeInfo.Category.PRIMITIVE) { // Only columns present in the batch and non-complex types. typeInfoList.add(inputObjInspectorsTypeInfos[i]); noNullsProjectionList.add(projectedColumn); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java index 69a2bef2dd3c54ed3696e1c342ad2197cb219df4..ac398ef83179a8aff07ba1ed78a5ddbf3e6145e1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/udf/VectorUDFArgDesc.java @@ -27,10 +27,10 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Descriptor for function argument. @@ -54,7 +54,7 @@ public VectorUDFArgDesc() { public void setConstant(ExprNodeConstantDesc expr) { isConstant = true; if (expr != null) { - if (expr.getTypeInfo().getCategory() == Category.PRIMITIVE) { + if (expr.getTypeInfo().getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) expr.getTypeInfo()) .getPrimitiveCategory(); if (primitiveCategory == PrimitiveCategory.VOID) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java index b90ce6ed2fb6a7e56d162dc1a63c49d9ca82089d..400c20bbafaa944f35b1cdd0b9cebaabe5d661bd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.io.RecordUpdater; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -43,6 +42,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.orc.OrcConf; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java index 9e476faeefafe2bc4adbe79daa6f6ef763fb64a1..0f8e14ad8ab86d47a5c7813725bbdb787f3898a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.Writable; @@ -270,7 +271,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } @Override @@ -374,7 +375,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.MAP; + return TypeInfo.Category.MAP; } @Override @@ -466,7 +467,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.LIST; + return TypeInfo.Category.LIST; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java index b87e00d00709b7b8bd8bff39d03f93980e5d86b5..ec2060cd5bb6b289888618a31db30b3fd6c58e3c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.orc.OrcProto; @@ -135,7 +136,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.UNION; + return TypeInfo.Category.UNION; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java index 302321c7f8111fb3f4e418e230c51e872cf7fa83..194fc71982024a360c5ed6732ed8794a79c60f41 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java @@ -17,7 +17,6 @@ import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -58,7 +57,7 @@ private static Type convertType(final String name, final TypeInfo typeInfo) { private static Type convertType(final String name, final TypeInfo typeInfo, final Repetition repetition) { - if (typeInfo.getCategory().equals(Category.PRIMITIVE)) { + if (typeInfo.getCategory().equals(TypeInfo.Category.PRIMITIVE)) { if (typeInfo.equals(TypeInfoFactory.stringTypeInfo)) { return Types.primitive(PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8) .named(name); @@ -107,13 +106,13 @@ private static Type convertType(final String name, final TypeInfo typeInfo, } else { throw new IllegalArgumentException("Unknown type: " + typeInfo); } - } else if (typeInfo.getCategory().equals(Category.LIST)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.LIST)) { return convertArrayType(name, (ListTypeInfo) typeInfo); - } else if (typeInfo.getCategory().equals(Category.STRUCT)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.STRUCT)) { return convertStructType(name, (StructTypeInfo) typeInfo); - } else if (typeInfo.getCategory().equals(Category.MAP)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.MAP)) { return convertMapType(name, (MapTypeInfo) typeInfo); - } else if (typeInfo.getCategory().equals(Category.UNION)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.UNION)) { throw new UnsupportedOperationException("Union type not implemented"); } else { throw new IllegalArgumentException("Unknown type: " + typeInfo); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java index bdffdf4543f30ad2def32f62a708b312f0170175..d2a075db6d38fde3cb7ee5d351a02b9fc2094176 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java @@ -18,7 +18,6 @@ import java.util.Map; import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.*; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; @@ -77,7 +76,7 @@ private void init(final GroupType selectedGroupType, converters = new Converter[selectedFieldCount]; this.repeatedConverters = new ArrayList(); - if (hiveTypeInfo != null && hiveTypeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) { + if (hiveTypeInfo != null && hiveTypeInfo.getCategory().equals(TypeInfo.Category.STRUCT)) { this.hiveFieldNames = ((StructTypeInfo) hiveTypeInfo).getAllStructFieldNames(); this.hiveFieldTypeInfos = ((StructTypeInfo) hiveTypeInfo).getAllStructFieldTypeInfos(); } @@ -126,9 +125,9 @@ private boolean isSubType( private TypeInfo getFieldTypeIgnoreCase(TypeInfo hiveTypeInfo, String fieldName, int fieldIndex) { if (hiveTypeInfo == null) { return null; - } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) { + } else if (hiveTypeInfo.getCategory().equals(TypeInfo.Category.STRUCT)) { return getStructFieldTypeInfo(fieldName, fieldIndex); - } else if (hiveTypeInfo.getCategory().equals(ObjectInspector.Category.MAP)) { + } else if (hiveTypeInfo.getCategory().equals(TypeInfo.Category.MAP)) { //This cover the case where hive table may have map but the data file is // of type array> //Using index in place of type name. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java index 7f2a684d28290b32bbbbcb531a18ee564bddeecf..0e96a3e12dbe38eb9dd732e77ccd11fab9edade9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.optimizer.FieldNode; import org.apache.hadoop.hive.ql.optimizer.NestedColumnFieldPruningUtils; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -146,7 +145,7 @@ private static Type getProjectedType(TypeInfo colType, Type fieldType) { .named(fieldType.getName()); case LIST: TypeInfo elemType = ((ListTypeInfo) colType).getListElementTypeInfo(); - if (elemType.getCategory() == ObjectInspector.Category.STRUCT) { + if (elemType.getCategory() == TypeInfo.Category.STRUCT) { Type subFieldType = fieldType.asGroupType().getType(0); if (!subFieldType.isPrimitive()) { String subFieldName = subFieldType.getName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java index 3f508bdcda0c22723b8854d4abfb16180aae93bb..70bace948a80449ea48ce00bf37e233497aebda8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/AbstractParquetMapInspector.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; @@ -39,7 +41,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.MAP; + return TypeInfo.Category.MAP; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java index d83376d3ffce7e7fb4989156cf2eddb0bfb09f05..20a0f451080f0378d5bf0dddac9ec2160d26b86e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ArrayWritableObjectInspector.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.ArrayWritable; @@ -107,12 +108,12 @@ private ObjectInspector getObjectInspector( return ParquetPrimitiveInspectorFactory.parquetStringInspector; } else if (typeInfo instanceof DecimalTypeInfo) { return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector((DecimalTypeInfo) typeInfo); - } else if (typeInfo.getCategory().equals(Category.STRUCT)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.STRUCT)) { return new ArrayWritableObjectInspector(false, (StructTypeInfo) typeInfo, (StructTypeInfo) prunedTypeInfo); - } else if (typeInfo.getCategory().equals(Category.LIST)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.LIST)) { final TypeInfo subTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); return new ParquetHiveArrayInspector(getObjectInspector(subTypeInfo, null)); - } else if (typeInfo.getCategory().equals(Category.MAP)) { + } else if (typeInfo.getCategory().equals(TypeInfo.Category.MAP)) { final TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); final TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); if (keyTypeInfo.equals(TypeInfoFactory.stringTypeInfo) || keyTypeInfo.equals(TypeInfoFactory.byteTypeInfo) @@ -145,7 +146,7 @@ private ObjectInspector getObjectInspector( @Override public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java index aec7423117c9f96f9358d127f1dbafb9f6bc4b8d..cf881e3a8ab738f4652a3216eb1e01c0907c5847 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveArrayInspector.java @@ -19,6 +19,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; @@ -42,7 +44,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.LIST; + return TypeInfo.Category.LIST; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java index e0018a5bc0a70044cd0679fe88be1ea730cd515a..fd82c4f1d3ed876a34f405b6c1d99bc8e6da857b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ParquetHiveRecord; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -160,7 +159,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { @Override public Writable serialize(final Object obj, final ObjectInspector objInspector) throws SerDeException { - if (!objInspector.getCategory().equals(Category.STRUCT)) { + if (!objInspector.getCategory().equals(TypeInfo.Category.STRUCT)) { throw new SerDeException("Cannot serialize " + objInspector.getCategory() + ". Can only serialize a struct"); } serializedSize = ((StructObjectInspector)objInspector).getAllStructFieldRefs().size(); @@ -254,7 +253,7 @@ private static void pruneFromSinglePath(PrunedStructTypeInfo prunedInfo, String this.selected = new boolean[typeInfo.getAllStructFieldTypeInfos().size()]; for (int i = 0; i < typeInfo.getAllStructFieldTypeInfos().size(); ++i) { TypeInfo ti = typeInfo.getAllStructFieldTypeInfos().get(i); - if (ti.getCategory() == Category.STRUCT) { + if (ti.getCategory() == TypeInfo.Category.STRUCT) { this.children.put(typeInfo.getAllStructFieldNames().get(i).toLowerCase(), new PrunedStructTypeInfo((StructTypeInfo) ti)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java index cf1210befc392f337bd96350939a0c744496db41..e22d4a62f98a9a5cb128df606e8903ae6f700748 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java @@ -13,6 +13,8 @@ */ package org.apache.hadoop.hive.ql.io.parquet.write; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -105,7 +107,7 @@ private MessageDataWriter createMessageWriter(StructObjectInspector inspector, G */ private DataWriter createWriter(ObjectInspector inspector, Type type) { if (type.isPrimitive()) { - checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE); + checkInspectorCategory(inspector, TypeInfo.Category.PRIMITIVE); PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector)inspector; switch (primitiveInspector.getPrimitiveCategory()) { case BOOLEAN: @@ -144,13 +146,13 @@ private DataWriter createWriter(ObjectInspector inspector, Type type) { OriginalType originalType = type.getOriginalType(); if (originalType != null && originalType.equals(OriginalType.LIST)) { - checkInspectorCategory(inspector, ObjectInspector.Category.LIST); + checkInspectorCategory(inspector, TypeInfo.Category.LIST); return new ListDataWriter((ListObjectInspector)inspector, groupType); } else if (originalType != null && originalType.equals(OriginalType.MAP)) { - checkInspectorCategory(inspector, ObjectInspector.Category.MAP); + checkInspectorCategory(inspector, TypeInfo.Category.MAP); return new MapDataWriter((MapObjectInspector)inspector, groupType); } else { - checkInspectorCategory(inspector, ObjectInspector.Category.STRUCT); + checkInspectorCategory(inspector, TypeInfo.Category.STRUCT); return new StructDataWriter((StructObjectInspector)inspector, groupType); } } @@ -162,7 +164,7 @@ private DataWriter createWriter(ObjectInspector inspector, Type type) { * @param category The category to match * @throws IllegalArgumentException if inspector does not match the category */ - private void checkInspectorCategory(ObjectInspector inspector, ObjectInspector.Category category) { + private void checkInspectorCategory(ObjectInspector inspector, Category category) { if (!inspector.getCategory().equals(category)) { throw new IllegalArgumentException("Invalid data type: expected " + category + " type, but found: " + inspector.getCategory()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java index 27fe828b7531584138cd002956a9fcc20f238f71..127991dddbae30212f09bd41220839a9ee07076e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; @@ -112,7 +111,7 @@ public SearchArgument buildSearchArgument() { */ private static BoxType getType(ExprNodeDesc expr) { TypeInfo type = expr.getTypeInfo(); - if (type.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (type.getCategory() == TypeInfo.Category.PRIMITIVE) { switch (((PrimitiveTypeInfo) type).getPrimitiveCategory()) { case BYTE: case SHORT: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java index c2a2fb103fc2107ce81af6b7aceaf897191c8062..aefe129aa2cda3c8653b618f6e2304bc985345bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import static org.apache.hadoop.hive.ql.optimizer.FieldNode.mergeFieldNodes; @@ -265,8 +264,8 @@ private static void checkListAndMap(ExprNodeDesc desc, FieldNode pathToRoot, Fie // Check cases for arr[i].f and map[key].v // For these we should not generate paths like arr.f or map.v // Otherwise we would have a mismatch between type info and path - if (ti.getCategory() != ObjectInspector.Category.LIST - && ti.getCategory() != ObjectInspector.Category.MAP) { + if (ti.getCategory() != TypeInfo.Category.LIST + && ti.getCategory() != TypeInfo.Category.MAP) { fn.addFieldNodes(pathToRoot); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java index 54d52f8c2a1ba21202d630037ab1c5a763a81115..d472dec2c71afc64361784c5b2a5df912a8cf64a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java @@ -63,7 +63,6 @@ import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; @@ -84,7 +83,6 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -912,7 +910,7 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List return null; } } - if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) { + if (constant.getTypeInfo().getCategory() != TypeInfo.Category.PRIMITIVE) { // nested complex types cannot be folded cleanly return null; } @@ -932,7 +930,7 @@ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List return null; } ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn; - if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) { + if (constant.getTypeInfo().getCategory() != TypeInfo.Category.PRIMITIVE) { // nested complex types cannot be folded cleanly return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java index e3dee932084c3d38c083b38a5a8a193f39ad7303..877ae4e09cc187612b093dc4265e9f580d798321 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java @@ -64,9 +64,9 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEpochMilli; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.parquet.Strings; import org.slf4j.Logger; @@ -229,7 +229,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, ExprNodeColumnDesc columnDesc = new ExprNodeColumnDesc(ci); descs.add(columnDesc); colNames.add(columnDesc.getExprString()); - if (columnDesc.getTypeInfo().getCategory() == ObjectInspector.Category.PRIMITIVE + if (columnDesc.getTypeInfo().getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo) columnDesc.getTypeInfo()).getPrimitiveCategory() == PrimitiveCategory.TIMESTAMPLOCALTZ) { if (timestampPos != -1) { throw new SemanticException("Multiple columns with timestamp with local time-zone type on query result; " diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java index 12af94e337b7fd62021ba76372bf216dc61e9b53..1516cc77ea44de4714d6a37f5e6ef2f5470f52e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java @@ -75,7 +75,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -151,7 +150,7 @@ public ExprNodeDesc visitFieldAccess(RexFieldAccess fieldAccess) { String child = fieldAccess.getField().getName(); TypeInfo parentType = parent.getTypeInfo(); // Allow accessing a field of list element structs directly from a list - boolean isList = (parentType.getCategory() == ObjectInspector.Category.LIST); + boolean isList = (parentType.getCategory() == TypeInfo.Category.LIST); if (isList) { parentType = ((ListTypeInfo) parentType).getListElementTypeInfo(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 39ff59129a86604a6e0054266e7a9878fdcde179..12f4afd68a9647f6dba791b53893c277718dfaf8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -95,7 +95,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -248,7 +247,7 @@ private RexNode convert(ExprNodeGenericFuncDesc func) throws SemanticException { GenericUDF tgtUdf = func.getGenericUDF(); boolean isNumeric = (tgtUdf instanceof GenericUDFBaseBinary - && func.getTypeInfo().getCategory() == Category.PRIMITIVE + && func.getTypeInfo().getCategory() == TypeInfo.Category.PRIMITIVE && (PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveTypeInfo) func.getTypeInfo()).getPrimitiveCategory()))); boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java index ea042bf1445b7cfbc25b2fe971328fee78652df7..d61cdfb00fbf9664a09b6a352fcee5bbdd67c129 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java @@ -51,8 +51,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -306,7 +306,7 @@ private Object handleDeterministicUdf(PcrExprProcCtx ctx, return new NodeInfoWrapper(WalkState.UNKNOWN, null, getOutExpr(fd, nodeOutputs)); } - if (has_part_col && fd.getTypeInfo().getCategory() == Category.PRIMITIVE) { + if (has_part_col && fd.getTypeInfo().getCategory() == TypeInfo.Category.PRIMITIVE) { // we need to evaluate result for every pruned partition if (fd.getTypeInfo().equals(TypeInfoFactory.booleanTypeInfo)) { // if the return type of the GenericUDF is boolean and all partitions agree on diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 190771ea6b1cbf4b669a8919271b25a689af941b..af5c1b00c387dbb5356356b8e6b24c19bd5cd732 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -30,17 +30,14 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.Stack; -import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; import org.apache.commons.lang.ArrayUtils; -import org.apache.calcite.util.Pair; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; import org.slf4j.Logger; @@ -81,16 +78,10 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc; import org.apache.hadoop.hive.ql.exec.vector.VectorColumnOutputMapping; import org.apache.hadoop.hive.ql.exec.vector.VectorColumnSourceMapping; -import org.apache.hadoop.hive.ql.exec.vector.VectorFileSinkOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOuterFilteredOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; import org.apache.hadoop.hive.ql.exec.vector.VectorizationOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil; import org.apache.hadoop.hive.ql.exec.vector.VectorizedUDAFs; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.HiveVectorAdaptorUsageMode; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType; @@ -101,22 +92,15 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; -import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; -import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; -import org.apache.hadoop.hive.ql.lib.GraphWalker; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; -import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; -import org.apache.hadoop.hive.ql.lib.PreOrderOnceWalker; -import org.apache.hadoop.hive.ql.lib.PreOrderWalker; import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType; import org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc; @@ -134,11 +118,9 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredWork; -import org.apache.hadoop.hive.ql.plan.OpTraits; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.ql.plan.Statistics; import org.apache.hadoop.hive.ql.plan.VectorAppMasterEventDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; import org.apache.hadoop.hive.ql.plan.VectorFileSinkDesc; @@ -175,13 +157,9 @@ import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc; import org.apache.hadoop.hive.ql.plan.VectorSelectDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc; -import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo; -import org.apache.hadoop.hive.ql.plan.VectorPartitionDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; @@ -227,8 +205,6 @@ import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear; import org.apache.hadoop.hive.ql.udf.UDFYear; import org.apache.hadoop.hive.ql.udf.generic.*; -import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; -import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.NullStructSerDe; @@ -236,7 +212,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -252,7 +228,6 @@ import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.common.util.AnnotationUtils; import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.StringUtils; import com.google.common.collect.ImmutableSet; import com.google.common.base.Preconditions; @@ -2559,7 +2534,7 @@ private boolean validatePTFOperator(PTFOperator op, VectorizationContext vContex TypeInfo typeInfo = exprNodeDesc.getTypeInfo(); Category category = typeInfo.getCategory(); boolean isSupportedType; - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { isSupportedType = false; } else { ColumnVector.Type colVecType = @@ -2664,7 +2639,7 @@ private boolean validateExprNodeDescRecursive(ExprNodeDesc desc, String expressi } if (desc.getChildren() != null) { if (isInExpression - && desc.getChildren().get(0).getTypeInfo().getCategory() == Category.STRUCT) { + && desc.getChildren().get(0).getTypeInfo().getCategory() == TypeInfo.Category.STRUCT) { // Don't restrict child expressions for projection. // Always use loose FILTER mode. if (!validateStructInExpression(desc, expressionTitle, VectorExpressionDescriptor.Mode.FILTER)) { @@ -2688,7 +2663,7 @@ private boolean validateStructInExpression(ExprNodeDesc desc, String expressionTitle, VectorExpressionDescriptor.Mode mode) { for (ExprNodeDesc d : desc.getChildren()) { TypeInfo typeInfo = d.getTypeInfo(); - if (typeInfo.getCategory() != Category.STRUCT) { + if (typeInfo.getCategory() != TypeInfo.Category.STRUCT) { return false; } StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo; @@ -2700,7 +2675,7 @@ private boolean validateStructInExpression(ExprNodeDesc desc, for (int f = 0; f < fieldCount; f++) { TypeInfo fieldTypeInfo = fieldTypeInfos.get(f); Category category = fieldTypeInfo.getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { setExpressionIssue(expressionTitle, "Cannot vectorize struct field " + fieldNames.get(f) + " of type " + fieldTypeInfo.getTypeName()); @@ -2788,7 +2763,7 @@ public static boolean validateDataType(String type, VectorExpressionDescriptor.M if (!result) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type); - if (typeInfo.getCategory() != Category.PRIMITIVE) { + if (typeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { if (allowComplex) { return true; } @@ -2808,7 +2783,7 @@ public static String getValidateDataTypeErrorMsg(String type, VectorExpressionDe if (!result) { TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type); - if (typeInfo.getCategory() != Category.PRIMITIVE) { + if (typeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { if (allowComplex && isVectorizationComplexTypesEnabled) { return null; } else if (!allowComplex) { @@ -3133,7 +3108,7 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi supportsKeyTypes = false; Category category = typeInfo.getCategory(); notSupportedKeyTypes.add( - (category != Category.PRIMITIVE ? category.toString() : + (category != TypeInfo.Category.PRIMITIVE ? category.toString() : ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory().toString())); } bigTableKeyTypeInfos[i] = typeInfo; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java index fe26283bd27d20890faeccdade1e24bd55f45fa2..8d1ed071e49297f1f6083c3a70065f456ccbfc38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsAutoGatherContext.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.plan.LoadFileDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -259,7 +258,7 @@ public static boolean canRunAutogatherStats(Operator curr) { for (ColumnInfo cinfo : curr.getSchema().getSignature()) { if (cinfo.getIsVirtualCol()) { return false; - } else if (cinfo.getObjectInspector().getCategory() != ObjectInspector.Category.PRIMITIVE) { + } else if (cinfo.getObjectInspector().getCategory() != TypeInfo.Category.PRIMITIVE) { return false; } else { switch (((PrimitiveTypeInfo) cinfo.getType()).getPrimitiveCategory()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index c97e2a979aeee4cc764310d1665d65e5a18f7474..9f1b0e926614bfb90bf12d210aaf85794b4fd90a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -228,7 +227,7 @@ private String getColTypeOf (String partKey) throws SemanticException{ if (colName.equalsIgnoreCase(col.getName())) { String type = col.getType(); TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type); - if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { logTypeWarning(colName, type); colNames.remove(colName); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java index d1854844eaebff40bf22f1634fce071c34ea9081..b7c65c356af8fd56dc70393a76caeb306bacedd5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java @@ -82,7 +82,6 @@ import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -592,7 +591,7 @@ static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException { private static void validateValueBoundaryExprType(ObjectInspector OI) throws SemanticException { - if (!OI.getCategory().equals(Category.PRIMITIVE)) { + if (!OI.getCategory().equals(TypeInfo.Category.PRIMITIVE)) { throw new SemanticException( String.format( "Value Boundary expression must be of primitive type. Found: %s", diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java index 89e84127a8aeb4a603df7a9cf7b7da2125a7949b..5f114d55117b7441560a43d443d6fd1f30d69e97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java @@ -43,9 +43,9 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -149,7 +149,7 @@ private ParseUtils() { public static ExprNodeDesc createConversionCast(ExprNodeDesc column, PrimitiveTypeInfo tableFieldTypeInfo) throws SemanticException { // Get base type, since type string may be parameterized - String baseType = TypeInfoUtils.getBaseName(tableFieldTypeInfo.getTypeName()); + String baseType = MetastoreTypeInfoUtils.getBaseName(tableFieldTypeInfo.getTypeName()); // If the type cast UDF is for a parameterized type, then it should implement // the SettableUDF interface so that we can pass in the params. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5c966538ee2e2d58c4b367cca649f2d8beade4f1..7075c7dbd4a3628fc418db43e3eb4ee3723a4a28 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -232,7 +232,6 @@ import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe2; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; @@ -7585,7 +7584,7 @@ Operator genConversionSelectOperator(String dest, QB qb, Operator input, if (!tableFieldTypeInfo.equals(rowFieldTypeInfo)) { // need to do some conversions here converted = true; - if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { + if (tableFieldTypeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { // cannot convert to complex types column = null; } else { @@ -7850,7 +7849,7 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo)) { // need to do some conversions here - if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) { + if (tableFieldTypeInfo.getCategory() != TypeInfo.Category.PRIMITIVE) { // cannot convert to complex types column = null; } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 14217e3978ffed81bb9103d27dc917cdd973eb4d..c826a2997cc6662961851fdc5cecea7274ff57f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -80,7 +80,6 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; @@ -935,11 +934,11 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, TypeInfo objectTypeInfo = object.getTypeInfo(); // Allow accessing a field of list element structs directly from a list - boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST); + boolean isList = (object.getTypeInfo().getCategory() == TypeInfo.Category.LIST); if (isList) { objectTypeInfo = ((ListTypeInfo) objectTypeInfo).getListElementTypeInfo(); } - if (objectTypeInfo.getCategory() != Category.STRUCT) { + if (objectTypeInfo.getCategory() != TypeInfo.Category.STRUCT) { throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr)); } TypeInfo t = ((StructTypeInfo) objectTypeInfo).getStructFieldTypeInfo(fieldNameString); @@ -958,7 +957,7 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, // Check whether this is a list or a map TypeInfo myt = children.get(0).getTypeInfo(); - if (myt.getCategory() == Category.LIST) { + if (myt.getCategory() == TypeInfo.Category.LIST) { // Only allow integer index for now if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), TypeInfoFactory.intTypeInfo)) { @@ -969,7 +968,7 @@ protected ExprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, // Calculate TypeInfo TypeInfo t = ((ListTypeInfo) myt).getListElementTypeInfo(); desc = new ExprNodeGenericFuncDesc(t, FunctionRegistry.getGenericUDFForIndex(), children); - } else if (myt.getCategory() == Category.MAP) { + } else if (myt.getCategory() == TypeInfo.Category.MAP) { if (!TypeInfoUtils.implicitConvertible(children.get(1).getTypeInfo(), ((MapTypeInfo) myt).getMapKeyTypeInfo())) { throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java index 73f449fc289e29746dfad1d90d2535d9e963d6a0..025ef6458dadf2c8267adaca8979da5cb27584a2 100755 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.common.StringInternUtils; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; @@ -122,9 +121,9 @@ private static String getFormatted(TypeInfo typeInfo, Object value) { @Override public String getExprString() { - if (typeInfo.getCategory() == Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { return getFormatted(typeInfo, value); - } else if (typeInfo.getCategory() == Category.STRUCT) { + } else if (typeInfo.getCategory() == TypeInfo.Category.STRUCT) { StringBuilder sb = new StringBuilder(); sb.append("const struct("); List items = (List) getWritableObjectInspector().getWritableConstantValue(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java index 86cc77d43b64182edac0f97f4f1c57eac45052d9..8e1bd6cfaa5889ce7af645e3c000565af7a23f49 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java @@ -25,17 +25,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationDesc; -import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hive.common.util.AnnotationUtils; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - /** * GroupByDesc. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPartitionConversion.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPartitionConversion.java index 0cfa646b36c613c4ce29463b8416f946d8cdaf07..98572d6f1c21f7a6f86096cda3e2319861eb976e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPartitionConversion.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorPartitionConversion.java @@ -19,7 +19,7 @@ package org.apache.hadoop.hive.ql.plan; import java.util.HashMap; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; + import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -82,8 +82,8 @@ public static boolean isImplicitVectorColumnConversion(TypeInfo fromTypeInfo, TypeInfo toTypeInfo) { - if (fromTypeInfo.getCategory() == Category.PRIMITIVE && - toTypeInfo.getCategory() == Category.PRIMITIVE) { + if (fromTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE && + toTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveCategory fromPrimitiveCategory = ((PrimitiveTypeInfo) fromTypeInfo).getPrimitiveCategory(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java index 85917e4f32d34a7e8187ba374a67e7581a8446b5..718377c32d7661aab7ff7efbc6c967b6cfaecf54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColStatsProcessor.java @@ -40,12 +40,11 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc; import org.apache.hadoop.hive.ql.plan.FetchWork; -import org.apache.hadoop.hive.ql.stats.ColumnStatisticsObjTranslator; import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; @@ -96,7 +95,7 @@ public int process(Hive db, Table tbl) throws Exception { List stats = new ArrayList(); InspectableObject packedRow; while ((packedRow = ftOp.getNextRow()) != null) { - if (packedRow.oi.getCategory() != ObjectInspector.Category.STRUCT) { + if (packedRow.oi.getCategory() != TypeInfo.Category.STRUCT) { throw new HiveException("Unexpected object type encountered while unpacking row"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java index 08cda4a1a8f17f1b5273b00f744354d0e7f7b892..90b8b89f1492e0ac7d50815bba69bd4060e9fe57 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/ColumnStatisticsObjTranslator.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; public class ColumnStatisticsObjTranslator { @@ -269,7 +270,7 @@ private static void unpackPrimitiveObject(ObjectInspector oi, Object o, String f } private static void unpackStructObject(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj cStatsObj) throws UnsupportedDoubleException { - if (oi.getCategory() != ObjectInspector.Category.STRUCT) { + if (oi.getCategory() != TypeInfo.Category.STRUCT) { throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString()); } @@ -283,7 +284,7 @@ private static void unpackStructObject(ObjectInspector oi, Object o, String fNam Object f = (list == null ? null : list.get(i)); String fieldName = fields.get(i).getFieldName(); - if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (foi.getCategory() == TypeInfo.Category.PRIMITIVE) { unpackPrimitiveObject(foi, f, fieldName, cStatsObj); } else { unpackStructObject(foi, f, fieldName, cStatsObj); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index cef87f59577af9ce354fe0d76b5c60fe29b2f76f..072bdba8d9da3c0663df020762902d6556fd6fe0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -76,7 +76,6 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector; @@ -1270,7 +1269,7 @@ public static long getSizeOfComplexTypes(HiveConf conf, ObjectInspector oi) { // check if list elements are primitive or Objects ObjectInspector leoi = scloi.getListElementObjectInspector(); - if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) { + if (leoi.getCategory().equals(TypeInfo.Category.PRIMITIVE)) { result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length, conf); } else { result += JavaDataModel.get().lengthForObjectArrayOfSize(length); @@ -1952,7 +1951,7 @@ public static long safeMult(long a, long b) { public static boolean hasDiscreteRange(ColStatistics colStat) { if (colStat.getRange() != null) { TypeInfo colType = TypeInfoUtils.getTypeInfoFromTypeString(colStat.getColumnType()); - if (colType.getCategory() == Category.PRIMITIVE) { + if (colType.getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveTypeInfo pti = (PrimitiveTypeInfo) colType; switch (pti.getPrimitiveCategory()) { case BOOLEAN: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java index aa5be09373d801e970ddb4daf9b35ca789a41b60..fc6b59a9d1e0e0f978f38bd9f458375eac546f6e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFEWAHBitmapBop.java @@ -32,11 +32,11 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.LongWritable; /** @@ -59,18 +59,18 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "The function " + name + "(b1, b2) takes exactly 2 arguments"); } - if (arguments[0].getCategory().equals(Category.LIST)) { + if (arguments[0].getCategory().equals(TypeInfo.Category.LIST)) { b1OI = (ListObjectInspector) arguments[0]; } else { throw new UDFArgumentTypeException(0, "\"" - + Category.LIST.toString().toLowerCase() + + TypeInfo.Category.LIST.toString().toLowerCase() + "\" is expected at function " + name + ", but \"" + arguments[0].getTypeName() + "\" is found"); } - if (!arguments[1].getCategory().equals(Category.LIST)) { + if (!arguments[1].getCategory().equals(TypeInfo.Category.LIST)) { throw new UDFArgumentTypeException(1, "\"" - + Category.LIST.toString().toLowerCase() + + TypeInfo.Category.LIST.toString().toLowerCase() + "\" is expected at function " + name + ", but \"" + arguments[1].getTypeName() + "\" is found"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java index 8f01dc7cb7dc86522edfe25f1768e6a08ae52223..8e4cc996eaf08680272154b61e72f13eae67b0b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDFReflect.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * common class for reflective UDFs @@ -46,7 +47,7 @@ void setupParameterOIs(ObjectInspector[] arguments, int start) throws UDFArgumen parameterClasses = new Class[length]; parameterJavaValues = new Object[length]; for (int i = 0; i < length; i++) { - if (arguments[i + start].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[i + start].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])" + " must be primitive (int, double, string, etc)."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java index d170d86f44fdbaef84a425a818f5c54b48d2da56..f2e8f79b283ffe68222430b6bf3dc9562cee8c9a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFAverage.java @@ -76,7 +76,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBinarySetFunctions.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBinarySetFunctions.java index 397ec92dab577d6fa4dfe3973362eed4488c30c6..ea80b04012c082d3315d1d3bc8ac3fa8a21b3872 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBinarySetFunctions.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFBinarySetFunctions.java @@ -412,12 +412,12 @@ private static void checkArgumentTypes(TypeInfo[] parameters) throws UDFArgument "Exactly two arguments are expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } - if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java index 226758981c0e55a8be1f85f18b23d6eab599a7b2..7e9952e084029c6d441acaa65217a0684010b6a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java @@ -71,7 +71,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly 2 (col + hll) or 3 (col + fm + #bitvectors) arguments are expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java index 2e7a5596bb2a0e7f1a55bec3865a16f814f12cf4..cdb8850a1ab2a11081017ad11e74db337ec2a8aa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java @@ -69,7 +69,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE // Validate the first parameter, which is the expression to compute over. This should be an // array of strings type, or an array of arrays of strings. PrimitiveTypeInfo pti; - if (parameters[0].getCategory() != ObjectInspector.Category.LIST) { + if (parameters[0].getCategory() != TypeInfo.Category.LIST) { throw new UDFArgumentTypeException(0, "Only list type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); @@ -100,9 +100,9 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // Validate the second parameter, which should be an array of strings - if(parameters[1].getCategory() != ObjectInspector.Category.LIST || + if(parameters[1].getCategory() != TypeInfo.Category.LIST || ((ListTypeInfo) parameters[1]).getListElementTypeInfo().getCategory() != - ObjectInspector.Category.PRIMITIVE) { + TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only arrays of strings are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); } @@ -113,7 +113,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // Validate the third parameter, which should be an integer to represent 'k' - if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[2].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(2, "Only integers are accepted but " + parameters[2].getTypeName() + " was passed as parameter 3."); } @@ -133,7 +133,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE // If the fourth parameter -- precision factor 'pf' -- has been specified, make sure it's // an integer. if(parameters.length == 4) { - if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[3].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(3, "Only integers are accepted but " + parameters[3].getTypeName() + " was passed as parameter 4."); } @@ -178,7 +178,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { outerInputOI = (ListObjectInspector) parameters[0]; if(outerInputOI.getListElementObjectInspector().getCategory() == - ObjectInspector.Category.LIST) { + TypeInfo.Category.LIST) { // We're dealing with input that is an array of arrays of strings innerInputOI = (StandardListObjectInspector) outerInputOI.getListElementObjectInspector(); inputOI = (PrimitiveObjectInspector) innerInputOI.getListElementObjectInspector(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java index d1517ab7ddd896a843ae8dce3776d05f22555b2f..d50c911acfbe2e6235fe58b4a316f6815e9e241d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCorrelation.java @@ -82,13 +82,13 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE "Exactly two arguments are expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } - if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java index 8b088f84051c019cbd020b8b242c11d3626c98da..7d490778c8b3c4b330839c83e0679fbe037298a2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java @@ -69,13 +69,13 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE "Exactly two arguments are expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } - if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java index 0dc391878fa51d6368bdac88d32120ed3614392e..251ea1e26a5f0cff535c712d3c8f435e531784a8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovarianceSample.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -50,13 +49,13 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly two arguments are expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); } - if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java index 4910fe10e85d2347bdf5071cc56d48621ddd113c..81cf2aa18ba526600203fdf0119b795c1dbfb66b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java @@ -69,7 +69,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // validate the first parameter, which is the expression to compute over - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); @@ -94,7 +94,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // validate the second parameter, which is the number of histogram bins - if (parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java index 568a7ec0ebe1ec9580de4e869031c9c862a37b1a..4bfa816e3e1dd96556c49bef84519df196ba985f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFLeadLag.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.udf.generic; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; @@ -57,7 +58,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo parameters) ObjectInspector amtOI = paramOIs[1]; if (!ObjectInspectorUtils.isConstantObjectInspector(amtOI) - || (amtOI.getCategory() != ObjectInspector.Category.PRIMITIVE) + || (amtOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector) amtOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.INT) { throw new UDFArgumentTypeException(1, fNm + " amount must be a integer value " + amtOI.getTypeName() + " was passed as parameter 1."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java index 450f7c25191c4b8f5f7a8f77b903f864fc07bdbf..700098cdd2168f5ae734fcf067afef6742c854f2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -91,7 +92,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S // Validate the first parameter, which is the expression to compute over. This should be a // numeric primitive type. - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); @@ -123,7 +124,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S case LIST: // An array was passed as parameter 2, make sure it's an array of primitives if(((ListObjectInspector) parameters[1]).getListElementObjectInspector().getCategory() != - ObjectInspector.Category.PRIMITIVE) { + TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "A floating point or decimal array argument may be passed as parameter 2, but " + parameters[1].getTypeName() + " was passed instead."); @@ -149,7 +150,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) throws S // If a third parameter has been specified, it should be an integer that specifies the number // of histogram bins to use in the percentile approximation. if(parameters.length == 3) { - if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[2].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(2, "Only a primitive argument is accepted as " + "parameter 3, but " + parameters[2].getTypeName() + " was passed instead."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java index 79b519cd030d700329ea0440428b1411dd8f6116..a401ce97f069a73cb30566b41d5478eaee092666 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStd.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -44,7 +43,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java index c68d77c57978fda427b235fd96a031b56ad17b8a..9c7210a395683bfaa848feec9e5065fba6f21c72 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStdSample.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -46,7 +45,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 1439b64e9720803696b84aac419a55021c27e38b..af2eaae52e027af8ec9b04ec02a08b4c8d84c5b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -69,7 +69,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); @@ -111,7 +111,7 @@ public GenericUDAFEvaluator getEvaluator(GenericUDAFParameterInfo info) } public static PrimitiveObjectInspector.PrimitiveCategory getReturnType(TypeInfo type) { - if (type.getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (type.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } switch (((PrimitiveTypeInfo) type).getPrimitiveCategory()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumEmptyIsZero.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumEmptyIsZero.java index 01c933c4a140f0b94614fd6e4a052cc1fb7a493f..55e3784216bd75321d3c7b1698a1f59f4559f040 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumEmptyIsZero.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSumEmptyIsZero.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -35,7 +34,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java index c9fb3df78617b20534dadbf73719208d11963bd1..cd658166d34bc15802319eadbfff752d6910bdb2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java @@ -158,7 +158,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java index 75f28b6690ff4797eb51ca69cbcabd07f8f25237..3fd83b289c7f473fe614b5f40d42749858c4f2b9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVarianceSample.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.*; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -47,7 +46,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) "Exactly one argument is expected."); } - if (parameters[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (parameters[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + parameters[0].getTypeName() + " is passed."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java index cbf3f3f8b9c225a5ff5478814bc50d454df0ea7b..e6a6c744a6680a5183584741e3430d311a09bf6f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java @@ -67,7 +67,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE // Validate the first parameter, which is the expression to compute over. This should be an // array of strings type, or an array of arrays of strings. PrimitiveTypeInfo pti; - if (parameters[0].getCategory() != ObjectInspector.Category.LIST) { + if (parameters[0].getCategory() != TypeInfo.Category.LIST) { throw new UDFArgumentTypeException(0, "Only list type arguments are accepted but " + parameters[0].getTypeName() + " was passed as parameter 1."); @@ -98,7 +98,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // Validate the second parameter, which should be an integer - if(parameters[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only integers are accepted but " + parameters[1].getTypeName() + " was passed as parameter 2."); } @@ -116,7 +116,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE } // Validate the third parameter, which should also be an integer - if(parameters[2].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[2].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(2, "Only integers are accepted but " + parameters[2].getTypeName() + " was passed as parameter 3."); } @@ -135,7 +135,7 @@ public GenericUDAFEvaluator getEvaluator(TypeInfo[] parameters) throws SemanticE // If we have the optional fourth parameter, make sure it's also an integer if(parameters.length == 4) { - if(parameters[3].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if(parameters[3].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(3, "Only integers are accepted but " + parameters[3].getTypeName() + " was passed as parameter 4."); } @@ -179,7 +179,7 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { outerInputOI = (ListObjectInspector) parameters[0]; if(outerInputOI.getListElementObjectInspector().getCategory() == - ObjectInspector.Category.LIST) { + TypeInfo.Category.LIST) { // We're dealing with input that is an array of arrays of strings innerInputOI = (StandardListObjectInspector) outerInputOI.getListElementObjectInspector(); inputOI = (PrimitiveObjectInspector) innerInputOI.getListElementObjectInspector(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java index 52c97c831557c328a833299850d6c6d1fa45b2ce..06dc3a93d2a38137c7d719bc045b9dc2e5eabcd2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java @@ -45,10 +45,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -296,8 +297,8 @@ protected void checkArgsSize(ObjectInspector[] arguments, int min, int max) protected void checkArgPrimitive(ObjectInspector[] arguments, int i) throws UDFArgumentTypeException { - ObjectInspector.Category oiCat = arguments[i].getCategory(); - if (oiCat != ObjectInspector.Category.PRIMITIVE) { + Category oiCat = arguments[i].getCategory(); + if (oiCat != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, getFuncName() + " only takes primitive types as " + getArgOrder(i) + " argument, got " + oiCat); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java index 5c459b2c0843a2d04d523ae43bdd9be8d8f2135e..bd90c1c9205b96c96a57438288609f28a868f709 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.udf.generic; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; @@ -30,14 +29,13 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -68,7 +66,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "ABS() requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "ABS only takes primitive types, got " + arguments[0].getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayContains.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayContains.java index 6be4c3c3ed4c6f694d9689e16575524d84c90715..e93825052f94b5ff69b0b61b1c2f720aa26301d9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayContains.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFArrayContains.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; /** @@ -62,7 +62,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) } // Check if ARRAY_IDX argument is of category LIST - if (!arguments[ARRAY_IDX].getCategory().equals(Category.LIST)) { + if (!arguments[ARRAY_IDX].getCategory().equals(TypeInfo.Category.LIST)) { throw new UDFArgumentTypeException(ARRAY_IDX, "\"" + org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "\" " + "expected at function ARRAY_CONTAINS, but " diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java index 8263ae84fb131ede03969dd086da19c30e8e3fe2..22b302dc3db815f8196462ba5b88e8655e7c4509 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAssertTrue.java @@ -24,13 +24,12 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.UDFType; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; /** @@ -53,7 +52,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throw new UDFArgumentLengthException( "ASSERT_TRUE() expects one argument."); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Argument to ASSERT_TRUE() should be primitive."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseArithmetic.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseArithmetic.java index 0735950a75272a8249fb790d50043c7f026a516d..6b1d05383d9546c33df1c4b0e10d7ebd1d072c77 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseArithmetic.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseArithmetic.java @@ -22,10 +22,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hive.common.HiveCompat; import org.apache.hive.common.HiveCompat.CompatLevel; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java index 97f4d1e9758abe7d9fcc66c0e46e7fe1ce1bd78f..f856a517183e75384a57d352efeb13e11b7cd76e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseCompare.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ReturnObjectInspectorResolver; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -78,11 +78,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java index a7bc6580e4d4a5658a62dcd93e59e614b1dc7474..7f99277cf3d1bb73a912b069521d329ae5aa4d0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java @@ -28,14 +28,13 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.io.ByteWritable; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -44,7 +43,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -90,11 +88,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < 2; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNwayCompare.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNwayCompare.java index d71f04f91b82f8a74e46f3af3e095951a3a50a3c..cea6496142e28cb9666ce4decb8fd3eaa4451b2a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNwayCompare.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNwayCompare.java @@ -51,7 +51,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException(getFuncName() + " requires at least 2 arguments, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException(getFuncName() + " only takes primitive types, got " + arguments[0].getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java index ce3c21a8134fb234f52bcfe63e36262c26f98024..be3a661b5181949a49cacfb0e49b153dc03856f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -88,7 +89,7 @@ protected abstract void performOp( // Convert input arguments to Text, if necessary. private Converter checkTextArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { - if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[i].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed."); } @@ -101,7 +102,7 @@ private Converter checkTextArguments(ObjectInspector[] arguments, int i) private Converter checkIntArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { - if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[i].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed."); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java index 043ef8c706896272a69f04a71d1975fce9c58a91..0fc0f01eb1105f375b4c2c73b8375bd8f9202bc7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseUnary.java @@ -28,12 +28,13 @@ import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; @@ -71,11 +72,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } Category category = arguments[0].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "The " + GenericUDFUtils.getOrdinal(1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCharacterLength.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCharacterLength.java index ce6cd301d72322cddc4a4b10e18ac5480652af9d..12118f7a3c9fd87559b2634d4cae0c4e56c6031b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCharacterLength.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCharacterLength.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; @@ -48,7 +49,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "CHARACTER_LENGTH requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "CHARACTER_LENGTH only takes primitive types, got " + argumentOI.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java index 62a7560eebd49c892d7d40bf0106396e8a4ff9ad..074f1e92fe4716f388457c275fe5c7c7afc7e6c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcat.java @@ -32,13 +32,13 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VarCharScalarConcatStringGroupCol; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.BytesWritable; @@ -82,7 +82,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen boolean fixedLengthReturnValue = true; int returnLength = 0; // Only for char/varchar return types for (int idx = 0; idx < arguments.length; ++idx) { - if (arguments[idx].getCategory() != Category.PRIMITIVE) { + if (arguments[idx].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException("CONCAT only takes primitive arguments"); } poi = (PrimitiveObjectInspector)arguments[idx]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java index ea9a59eeb1d23ef5b5a730c7230491ef6a89df05..f79e4ab5e19e238d6078a8c99243d064b88d7cfc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFConcatWS.java @@ -26,13 +26,12 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Text; /** @@ -86,7 +85,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } protected boolean isStringOrVoidType(ObjectInspector oi) { - if (oi.getCategory() == Category.PRIMITIVE) { + if (oi.getCategory() == TypeInfo.Category.PRIMITIVE) { if (PrimitiveGrouping.STRING_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveObjectInspector) oi).getPrimitiveCategory()) @@ -116,7 +115,7 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { } else { sb.append(separator); } - if (argumentOIs[i].getCategory().equals(Category.LIST)) { + if (argumentOIs[i].getCategory().equals(TypeInfo.Category.LIST)) { Object strArray = arguments[i].get(); ListObjectInspector strArrayOI = (ListObjectInspector) argumentOIs[i]; boolean strArrayFirst = true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java index b73893d0bcd538942f44ecdd2250d7336fe9e884..c88521f3a27d472b428382301f7329f720b52e0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDate.java @@ -32,12 +32,12 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hive.common.util.DateParser; /** @@ -66,7 +66,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException( "to_date() requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException("to_date() only accepts STRING/TIMESTAMP/DATEWRITABLE types, got " + arguments[0].getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java index 8ba103beb6af9dec84a6eaeb3efb32b763d77c0c..9528e087a439eeadd20bbb9806a2ebf873cdf98b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateAdd.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hive.common.util.DateParser; @@ -79,12 +80,12 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException( "date_add() requires 2 argument, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + arguments[0].getTypeName() + " is passed. as first arguments"); } - if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + arguments[1].getTypeName() + " is passed. as second arguments"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java index e777a8fe83f5c47ea7d554d2104c6ac6bbbf3cc5..2cbff3ad009f68d2e8f505b7a46455075b14a125 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDateDiff.java @@ -21,7 +21,6 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.sql.Date; -import java.util.TimeZone; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; @@ -41,6 +40,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; /** @@ -134,7 +134,7 @@ private Date convertToDate(PrimitiveCategory inputType, Converter converter, Def } private Converter checkArguments(ObjectInspector[] arguments, int i) throws UDFArgumentException { - if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[i].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + arguments[i].getTypeName() + " is passed. as first arguments"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java index e5bb42f77ffbe3d671b4f6f08941561889f747c6..859c02e235d0cc41204a48bff2c3248f80b9689c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java @@ -30,10 +30,8 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; @@ -41,6 +39,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @Description(name = "decode", value = "_FUNC_(bin, str) - Decode the first argument using the second argument character set", @@ -58,7 +57,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException("Decode() requires exactly two arguments"); } - if (arguments[0].getCategory() != Category.PRIMITIVE){ + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE){ throw new UDFArgumentTypeException(0, "The first argument to Decode() must be primitive"); } @@ -72,7 +71,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentTypeException(0, "The first argument to Decode() must be binary"); } - if (arguments[1].getCategory() != Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "The second argument to Decode() must be primitive"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java index aab6e823734918f1f0f3e6d0f9f009f0ad632346..dd9ce3891bd05d17aa1ef2e4b0a4e9eaa4496700 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEWAHBitmapEmpty.java @@ -31,11 +31,11 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.LongWritable; @@ -52,11 +52,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "The function EWAH_BITMAP_EMPTY(b) takes exactly 1 argument"); } - if (arguments[0].getCategory().equals(Category.LIST)) { + if (arguments[0].getCategory().equals(TypeInfo.Category.LIST)) { bitmapOI = (ListObjectInspector) arguments[0]; } else { throw new UDFArgumentTypeException(0, "\"" - + Category.LIST.toString().toLowerCase() + + TypeInfo.Category.LIST.toString().toLowerCase() + "\" is expected at function EWAH_BITMAP_EMPTY, but \"" + arguments[0].getTypeName() + "\" is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFElt.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFElt.java index 0203a31cc10af6236bd6547e7766b5e525cca80f..d5f4681df72c3071084b5fb3d4771a78a9337008 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFElt.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFElt.java @@ -24,9 +24,10 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; /** @@ -52,11 +53,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function ELT is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java index 1aa771dfcad49a70566caed00cc76406b738036f..83563043b1a3e7f811792c2c2a147d471bc97376 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java @@ -30,14 +30,13 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; @Description(name = "encode", @@ -57,7 +56,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen throw new UDFArgumentLengthException("Encode() requires exactly two arguments"); } - if (arguments[0].getCategory() != Category.PRIMITIVE || + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE || PrimitiveGrouping.STRING_GROUP != PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveObjectInspector)arguments[0]).getPrimitiveCategory())){ throw new UDFArgumentTypeException( @@ -66,7 +65,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen stringOI = (PrimitiveObjectInspector) arguments[0]; - if (arguments[1].getCategory() != Category.PRIMITIVE || + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE || PrimitiveGrouping.STRING_GROUP != PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveObjectInspector)arguments[1]).getPrimitiveCategory())){ throw new UDFArgumentTypeException( diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java index 4783c5baba5b8a197daf62398d2bfd1c2104022a..e9ea2818c7b7813e41fae2d26fcb6432242fefae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFExtractUnion.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java index c5f2a51a95ebadebbbcf63d76bbaba3737cd1a4e..63842a625db7039b36f4cf7ca509bd2b54b59afc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFField.java @@ -18,15 +18,15 @@ package org.apache.hadoop.hive.ql.udf.generic; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; /** @@ -51,11 +51,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function FIELD is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java index f90819c354d0e41b9804af8af32e098a8aa4a95a..ad8b78fdb4ebfd4faa9be0ee22e4706e49dee36a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloorCeilBase.java @@ -25,13 +25,14 @@ import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.LongWritable; @@ -58,11 +59,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } Category category = arguments[0].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "The " + GenericUDFUtils.getOrdinal(1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java index d11e51b3cbf079eadb9015f372e7a0efd8c78103..b68b632552a19ed74eba6efc80b49a64b7ac1aba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGrouping.java @@ -25,13 +25,11 @@ import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantLongObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.LongWritable; import com.google.common.math.LongMath; @@ -57,7 +55,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "grouping() requires at least 2 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "The first argument to grouping() must be primitive"); } PrimitiveObjectInspector arg1OI = (PrimitiveObjectInspector) arguments[0]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java index 90ff765ad7912b00c85008f30ed2239220cb7075..02e3c81f3cae0900d4563bb62ca4c69b86a54ba3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIf.java @@ -62,8 +62,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprStringScalarVarCharScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprCharScalarStringScalar; import org.apache.hadoop.hive.ql.exec.vector.expressions.IfExprVarCharScalarStringScalar; - - +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * IF(expr1,expr2,expr3)
@@ -113,7 +112,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "The function IF(expr1,expr2,expr3) accepts exactly 3 arguments."); } - boolean conditionTypeIsOk = (arguments[0].getCategory() == ObjectInspector.Category.PRIMITIVE); + boolean conditionTypeIsOk = (arguments[0].getCategory() == TypeInfo.Category.PRIMITIVE); if (conditionTypeIsOk) { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) arguments[0]); conditionTypeIsOk = (poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java index cf26fce00f0b03e10cd6c7ea28b6c403649bb651..4b70eab600e5f74ecde35f5560a0f8b7c9c35430 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java @@ -34,10 +34,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; -import com.esotericsoftware.minlog.Log; - /** * GenericUDFIn * @@ -121,7 +120,7 @@ private void checkIfInSetConstant(){ // (and those from IN(...) follow it) private void prepareInSet(DeferredObject[] arguments) throws HiveException { constantInSet = new HashSet(); - if (compareOI.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) { + if (compareOI.getCategory().equals(TypeInfo.Category.PRIMITIVE)) { for (int i = 1; i < arguments.length; ++i) { constantInSet.add(((PrimitiveObjectInspector) compareOI) .getPrimitiveJavaObject(conversionHelper diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInBloomFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInBloomFilter.java index d739af94f27cf7bbaa13cf82a1c77f5211b08b1f..4669c754d4916ea79c7ca54c991bbde8d904cd2b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInBloomFilter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInBloomFilter.java @@ -32,12 +32,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hive.common.util.BloomKFilter; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.sql.Timestamp; @@ -62,7 +62,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } // Verify individual arguments - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "The 1st argument must be a primitive type but " + arguments[0].getTypeName() + " was passed"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java index 4c0cb2b739bc78f6d6992348bd100220134c2b00..bb727961de6db96cf5cac7c8e3d4bda57895de9f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIndex.java @@ -37,8 +37,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; /** @@ -73,8 +73,8 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen mapOI = null; } else { throw new UDFArgumentTypeException(0, "\"" - + Category.MAP.toString().toLowerCase() + "\" or \"" - + Category.LIST.toString().toLowerCase() + + TypeInfo.Category.MAP.toString().toLowerCase() + "\" or \"" + + TypeInfo.Category.LIST.toString().toLowerCase() + "\" is expected at function INDEX, but \"" + arguments[0].getTypeName() + "\" is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java index a5579b9a9c3bdeef0f580a6e4636b624e93baa00..7c8f1bf8a5925b41fd71813723aad9f3ef09c834 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java @@ -24,9 +24,10 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -58,11 +59,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function INSTR is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInternalInterval.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInternalInterval.java index 4da3f18386765bb8a1c0b3610022ec37671549af..4c4bdf6e8a2d83a59c4b62d4de054950d1b5772d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInternalInterval.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInternalInterval.java @@ -33,13 +33,13 @@ import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hive.common.util.DateUtils; @@ -76,7 +76,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } // check value argument - if (arguments[1].getCategory() != Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "The first argument to " + getFuncName() + " must be primitive"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java index 58c80d1544d96182623c7bf422e7ae70b1013e71..f06bdacc49253a06904c57b177e2b4fd424f4bdb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLeadLag.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; public abstract class GenericUDFLeadLag extends GenericUDF { @@ -83,7 +84,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen if (arguments.length > 1) { ObjectInspector amtOI = arguments[1]; if (!ObjectInspectorUtils.isConstantObjectInspector(amtOI) - || (amtOI.getCategory() != ObjectInspector.Category.PRIMITIVE) + || (amtOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector) amtOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.INT) { throw new UDFArgumentTypeException(1, _getFnName() + " amount must be a integer value " + amtOI.getTypeName() + " was passed as parameter 1."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLength.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLength.java index f4ac350b71af78cd3d7da414c3dabdc27a15167b..60f43cd385165fa83106b92658b819532e847b4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLength.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLength.java @@ -23,11 +23,11 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLength; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.serde2.lazy.LazyBinary; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; @@ -54,7 +54,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "LENGTH requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "LENGTH only takes primitive types, got " + argumentOI.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java index cebc7524e828598326531d12f59962ff7bf34ec1..8e9fee4fbba7bfc58ad96e7a8947899d0333b921 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java @@ -24,9 +24,10 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -59,11 +60,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < arguments.length; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of function LOCATE is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java index 4913be834c49c83f50c90c3523f6ccf40cecf6cb..c91975ba59f42b183f53d00c392d523610522f3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLower.java @@ -25,13 +25,13 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringLower; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; /** @@ -56,7 +56,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "LOWER requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "LOWER only takes primitive types, got " + argumentOI.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapKeys.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapKeys.java index 27c88fffe4987fd4ba42347698d5a2b217158541..f0e1623cc65e60b02a3ee3c84d710ea92232283d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapKeys.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapKeys.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * GenericUDFMapKeys. @@ -48,7 +48,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throw new UDFArgumentLengthException("The function MAP_KEYS only accepts one argument."); } else if (!(arguments[0] instanceof MapObjectInspector)) { throw new UDFArgumentTypeException(0, "\"" - + Category.MAP.toString().toLowerCase() + + TypeInfo.Category.MAP.toString().toLowerCase() + "\" is expected at function MAP_KEYS, " + "but \"" + arguments[0].getTypeName() + "\" is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapValues.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapValues.java index 670b90b62a647d461de34d9ef81b1f151de82f31..f047b560f154199014620c871ae181a9b7101e89 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapValues.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMapValues.java @@ -28,8 +28,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * GenericUDFMapValues. @@ -48,7 +48,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throw new UDFArgumentLengthException("The function MAP_VALUES only accepts 1 argument."); } else if (!(arguments[0] instanceof MapObjectInspector)) { throw new UDFArgumentTypeException(0, "\"" - + Category.MAP.toString().toLowerCase() + + TypeInfo.Category.MAP.toString().toLowerCase() + "\" is expected at function MAP_VALUES, " + "but \"" + arguments[0].getTypeName() + "\" is found"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIMinus.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIMinus.java index e1673b2c9f80f1af29e217f4469e668222eab44b..268e7940ea82529afb27a81ea6c22c702dc9c6eb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIMinus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIMinus.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -91,11 +91,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) for (int i = 0; i < 2; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIPlus.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIPlus.java index a57b373983435a174c1e3a6ae6cb3ccd70dbc185..77870694f7beee64dbaf9299c8cf54388b75156b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIPlus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDTIPlus.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -88,11 +88,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) for (int i = 0; i < 2; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOctetLength.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOctetLength.java index 825066fc02cd82c1d989f407cd89e299098ec74a..c1dfa9f602ce5ef264355f11af0d2044edb7f2ac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOctetLength.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOctetLength.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.IntWritable; @@ -48,7 +49,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "OCTET_LENGTH requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "OCTET_LENGTH only takes primitive types, got " + argumentOI.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java index 91f730a8cc953dbfa873c911df795f0bfba2fd91..12e31283b6d7084e145c6f8ae8adf7b0119b021a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPower.java @@ -28,11 +28,12 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.io.DoubleWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @Description(name = "power,pow", value = "_FUNC_(x1, x2) - raise x1 to the power of x2", @@ -64,11 +65,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen for (int i = 0; i < 2; i++) { Category category = arguments[i].getCategory(); - if (category != Category.PRIMITIVE) { + if (category != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "The " + GenericUDFUtils.getOrdinal(i + 1) + " argument of " + opName + " is expected to a " - + Category.PRIMITIVE.toString().toLowerCase() + " type, but " + + TypeInfo.Category.PRIMITIVE.toString().toLowerCase() + " type, but " + category.toString().toLowerCase() + " is found"); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java index e57c3f74e22c4d8e07ef18001b50a7b94602d414..55c38568ecbde9b7947f3f67411301a8dc053eda 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java @@ -21,7 +21,6 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Formatter; -import java.util.IllegalFormatConversionException; import java.util.Locale; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -33,12 +32,11 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Text; /** @@ -69,7 +67,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen WritableStringObjectInspector resultOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector; - if (arguments[0].getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) arguments[0]); if (poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.STRING || poi.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR || @@ -88,9 +86,9 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } for (int i = 1; i < arguments.length; i++) { - if (!arguments[i].getCategory().equals(Category.PRIMITIVE)){ + if (!arguments[i].getCategory().equals(TypeInfo.Category.PRIMITIVE)){ throw new UDFArgumentTypeException(i, "Argument " + (i + 1) - + " of function PRINTF must be \"" + Category.PRIMITIVE + + " of function PRINTF must be \"" + TypeInfo.Category.PRIMITIVE + "\", but \"" + arguments[i].getTypeName() + "\" was found."); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java index f0fcf69856187d10bd28cb512e98180d51c05158..0b6602215253df6bf5ff28acb0ccdef3e8404660 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect2.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -71,7 +72,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "The function GenericUDFReflect2(arg0,method[,arg1[,arg2]...])" + " accepts 2 or more arguments."); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "The target instance should be a primitive type."); } targetOI = (PrimitiveObjectInspector) arguments[0]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java index ea7466ea408e08f8c07ae16d96dbb980eeaa1ab4..efd751b68ce44999654a6981ebde1a23effe26d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -45,6 +44,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantLongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantShortObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; @@ -84,14 +84,14 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "ROUND requires one or two argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "ROUND input only takes primitive types, got " + arguments[0].getTypeName()); } inputOI = (PrimitiveObjectInspector) arguments[0]; if (arguments.length == 2) { - if (arguments[1].getCategory() != Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "ROUND second argument only takes primitive types, got " + arguments[1].getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java index 10763a1640b67da36df33c5f05862b4a54f43030..00fec188c14c363574b9d9f6458e00f2a033f29a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSize.java @@ -27,8 +27,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.IntWritable; /** @@ -49,11 +50,11 @@ public ObjectInspector initialize(ObjectInspector[] arguments) } Category category = arguments[0].getCategory(); String typeName = arguments[0].getTypeName(); - if (category != Category.MAP && category != Category.LIST + if (category != TypeInfo.Category.MAP && category != TypeInfo.Category.LIST && !typeName.equals(serdeConstants.VOID_TYPE_NAME)) { throw new UDFArgumentTypeException(0, "\"" - + Category.MAP.toString().toLowerCase() + "\" or \"" - + Category.LIST.toString().toLowerCase() + + TypeInfo.Category.MAP.toString().toLowerCase() + "\" or \"" + + TypeInfo.Category.LIST.toString().toLowerCase() + "\" is expected at function SIZE, " + "but \"" + arguments[0].getTypeName() + "\" is found"); } @@ -65,9 +66,9 @@ public ObjectInspector initialize(ObjectInspector[] arguments) @Override public Object evaluate(DeferredObject[] arguments) throws HiveException { Object data = arguments[0].get(); - if (returnOI.getCategory() == Category.MAP) { + if (returnOI.getCategory() == TypeInfo.Category.MAP) { result.set(((MapObjectInspector) returnOI).getMapSize(data)); - } else if (returnOI.getCategory() == Category.LIST) { + } else if (returnOI.getCategory() == TypeInfo.Category.LIST) { result.set(((ListObjectInspector) returnOI).getListLength(data)); } else if (returnOI.getTypeName().equals(serdeConstants.VOID_TYPE_NAME)) { // null diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java index 3e2d8864f94768578291367d16e4939e69abbab4..0f0e73c98d963cd9291ba8ba2c927ccadea7277d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSortArray.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * Generic UDF for array sort @@ -63,7 +64,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen switch(arguments[0].getCategory()) { case LIST: if(!((ListObjectInspector)(arguments[0])).getListElementObjectInspector() - .getCategory().equals(ObjectInspector.Category.UNION)) { + .getCategory().equals(TypeInfo.Category.UNION)) { break; } default: diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java index b42b304b9bcbe1e602b9b9b667be8dd13d7fbdc6..91dc37b5c77e3047da3f091c4aff1e49690e97a9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSoundex.java @@ -31,6 +31,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.Text; /** @@ -90,8 +92,8 @@ public String getDisplayString(String[] children) { protected void checkIfPrimitive(ObjectInspector[] arguments, int i, String argOrder) throws UDFArgumentTypeException { - ObjectInspector.Category oiCat = arguments[i].getCategory(); - if (oiCat != ObjectInspector.Category.PRIMITIVE) { + Category oiCat = arguments[i].getCategory(); + if (oiCat != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, getFuncName() + " only takes primitive types as " + argOrder + " argument, got " + oiCat); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java index 480ffdef0e865844c22e8fdc21e05405fb6c1d71..2d2ab852a32dca1ee89b8c866c460b17e36e3a91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStringToMap.java @@ -26,15 +26,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * GenericUDFStringToMap. @@ -56,7 +53,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException { for (int idx = 0; idx < Math.min(arguments.length, 3); ++idx) { - if (arguments[idx].getCategory() != Category.PRIMITIVE + if (arguments[idx].getCategory() != TypeInfo.Category.PRIMITIVE || PrimitiveObjectInspectorUtils.getPrimitiveGrouping( ((PrimitiveObjectInspector) arguments[idx]).getPrimitiveCategory()) != PrimitiveGrouping.STRING_GROUP) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java index f6fed9a28469bb6764e55cab2659aed46b69993a..2947b659747c89cf30f06e7c7f3521a2a6505b46 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFStruct.java @@ -26,8 +26,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @Description(name = "struct", value = "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values") @@ -48,7 +48,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) boolean constantStruct = true; for (int i = 0; i < arguments.length; i++) { ObjectInspector oi = arguments[i]; - constantStruct &= (oi.getCategory() == Category.PRIMITIVE) + constantStruct &= (oi.getCategory() == TypeInfo.Category.PRIMITIVE) && (oi instanceof ConstantObjectInspector); if (constantStruct) { // nested complex types trigger Kryo issue #216 in plan deserialization diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java index 772967f23c7516671d41f1be569b5d185f48c458..0bd22915d240cce12e32fd16730f59691bacfdf8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java @@ -32,18 +32,16 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorUDFUnixTimeStampTimestamp; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; /** * deterministic version of UDFUnixTimeStamp. enforces argument @@ -74,7 +72,7 @@ protected void initializeInput(ObjectInspector[] arguments) throws UDFArgumentEx "requires at least one argument"); } for (ObjectInspector argument : arguments) { - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException(getName().toUpperCase() + " only takes string/date/timestamp types, got " + argument.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java index 4d75c6d810a81b2fe6d4b30cb8ee1d48e131f828..bca8d2514b64ef9433849d6ba282a07233e19b68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java @@ -31,11 +31,11 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.Text; /** @@ -135,7 +135,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen } for (int i = 0; i < arguments.length; i++) { - if (arguments[i].getCategory() != Category.PRIMITIVE) { + if (arguments[i].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(i, "A string argument was expected but an argument of type " + arguments[i].getTypeName() + " was given."); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java index fa245e7847d6723a58e75c8dc137d403f83b7e6b..b5cd905563afaec30d11c5f5c9ef4657ddf8cb91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTrunc.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -49,6 +48,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -141,14 +141,14 @@ private ObjectInspector initializeNumber(ObjectInspector[] arguments) "TRUNC requires one or two argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "TRUNC input only takes primitive types, got " + arguments[0].getTypeName()); } inputOI = (PrimitiveObjectInspector) arguments[0]; if (arguments.length == 2) { - if (arguments[1].getCategory() != Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "TRUNC second argument only takes primitive types, got " + arguments[1].getTypeName()); } @@ -218,12 +218,12 @@ private ObjectInspector initializeDate(ObjectInspector[] arguments) throw new UDFArgumentLengthException("trunc() requires 2 argument, got " + arguments.length); } - if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " + arguments[0].getTypeName() + " is passed. as first arguments"); } - if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) { + if (arguments[1].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentTypeException(1, "Only primitive type arguments are accepted but " + arguments[1].getTypeName() + " is passed. as second arguments"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java index b263e46ba280d6bdaae19f2ac826b3dc185f0d7a..ddf5283d55c97b304d2a8e0b953731d9af3b6b5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUpper.java @@ -25,13 +25,13 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.StringUpper; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.StringConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; /** @@ -56,7 +56,7 @@ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumen "UPPER requires 1 argument, got " + arguments.length); } - if (arguments[0].getCategory() != Category.PRIMITIVE) { + if (arguments[0].getCategory() != TypeInfo.Category.PRIMITIVE) { throw new UDFArgumentException( "UPPER only takes primitive types, got " + argumentOI.getTypeName()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java index c91865b17357ff0015edac55be694097c7c6e62b..2a2d729bc35e5bcf6c00ec3edb3661429e5bcda6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.IdentityConverter; @@ -253,7 +252,7 @@ public static TypeInfo deriveInType(List children) { TypeInfo returnType = null; for (ExprNodeDesc node : children) { TypeInfo ti = node.getTypeInfo(); - if (ti.getCategory() == Category.PRIMITIVE + if (ti.getCategory() == TypeInfo.Category.PRIMITIVE && ((PrimitiveTypeInfo)ti).getPrimitiveCategory() == PrimitiveCategory.VOID) { continue; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java index 573d8380d35867e19f0f9479e907cafb1ab53ad5..9e9a6acedaf372abbd37503d6ac5d1641d97ceee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFInline.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.udf.generic; -import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hive.ql.exec.Description; @@ -26,8 +25,8 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @Description(name ="inline", value= "_FUNC_( ARRAY( STRUCT()[,STRUCT()] " + "- explodes and array and struct into a table") @@ -44,13 +43,13 @@ public StructObjectInspector initialize(ObjectInspector[] ois) throws UDFArgumen if (ois.length!=1){ throw new UDFArgumentException("UDF tables only one argument"); } - if (ois[0].getCategory()!= Category.LIST){ + if (ois[0].getCategory()!= TypeInfo.Category.LIST){ throw new UDFArgumentException("Top level object must be an array but " + "was "+ois[0].getTypeName()); } li = (ListObjectInspector) ois[0]; ObjectInspector sub=li.getListElementObjectInspector(); - if (sub.getCategory() != Category.STRUCT){ + if (sub.getCategory() != TypeInfo.Category.STRUCT){ throw new UDFArgumentException("The sub element must be struct, but was "+sub.getTypeName()); } return (StructObjectInspector) sub; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java index 82feca293b00000e2a00ae576dffd4478243a36f..6a596f0be90cca61ab5c65b18291b674494ea77f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFJSONTuple.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -113,7 +114,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) } for (int i = 0; i < args.length; ++i) { - if (args[i].getCategory() != ObjectInspector.Category.PRIMITIVE || + if (args[i].getCategory() != TypeInfo.Category.PRIMITIVE || !args[i].getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { throw new UDFArgumentException("json_tuple()'s arguments have to be string type"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java index f3c6ca8a1e4d2e052c1448ef191ccb868f1f6da3..24de3c8605afcb48c7bdd64f66abed739c0e396c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFParseUrlTuple.java @@ -24,6 +24,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Description; @@ -90,7 +91,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) } for (int i = 0; i < args.length; ++i) { - if (args[i].getCategory() != ObjectInspector.Category.PRIMITIVE || + if (args[i].getCategory() != TypeInfo.Category.PRIMITIVE || !args[i].getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { throw new UDFArgumentException("parse_url_tuple()'s arguments have to be string type"); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java index fde76475e76d4947ad75abb545fc47f45ee189e9..3eed633eb45731dd109815a02e192c40cf8b1ac9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFPosExplode.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * PosExplode. @@ -54,7 +55,7 @@ public StructObjectInspector initialize(ObjectInspector[] args) throws UDFArgume throw new UDFArgumentException("posexplode() takes only one argument"); } - if (args[0].getCategory() != ObjectInspector.Category.LIST) { + if (args[0].getCategory() != TypeInfo.Category.LIST) { throw new UDFArgumentException("posexplode() takes an array as a parameter"); } listOI = (ListObjectInspector) args[0]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java index 8f3dfdbe3c1db7cc018b11de5ec11a2e10df9456..b16243c27953a33c837e77b87597b5cf23c86c06 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/MatchPath.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; /** @@ -219,7 +220,7 @@ private void validateAndSetupPatternStr(MatchPath evaluator, ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolPatternArgOI) || - (symbolPatternArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + (symbolPatternArgOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector)symbolPatternArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) { @@ -250,7 +251,7 @@ private void validateAndSetupSymbolInfo(MatchPath evaluator, ObjectInspector symbolNameArgOI = symbolNameArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(symbolNameArgOI) || - (symbolNameArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + (symbolNameArgOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector)symbolNameArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) { @@ -263,7 +264,7 @@ private void validateAndSetupSymbolInfo(MatchPath evaluator, PTFExpressionDef symolExprArg = args.get(i+1); ObjectInspector symolExprArgOI = symolExprArg.getOI(); - if ( (symolExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + if ( (symolExprArgOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector)symolExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN ) { @@ -284,7 +285,7 @@ private void validateAndSetupResultExprStr(MatchPath evaluator, ObjectInspector resultExprArgOI = resultExprArg.getOI(); if ( !ObjectInspectorUtils.isConstantObjectInspector(resultExprArgOI) || - (resultExprArgOI.getCategory() != ObjectInspector.Category.PRIMITIVE) || + (resultExprArgOI.getCategory() != TypeInfo.Category.PRIMITIVE) || ((PrimitiveObjectInspector)resultExprArgOI).getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.STRING ) { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java index 83cdb2d79cf565724a7a2495686a6f7ca9863257..70a87b3d401c1c5a8ba96c69493efe49fb034c6b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Properties; import java.util.Random; @@ -38,10 +37,8 @@ import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead; import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite; -import org.apache.hadoop.hive.serde2.lazy.fast.StringToDouble; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; @@ -65,7 +62,7 @@ private void verifyRead( DeserializeRead deserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VectorVerifyFast.verifyDeserializeRead(deserializeRead, typeInfo, expectedObject); } else { Object complexFieldObj = VectorVerifyFast.deserializeReadComplexType(deserializeRead, typeInfo); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java index 3f993284f19e8d87d3e99dc4dbc4d43da2dcd788..38a4ea6be3be3de9e10447a125e637defc8f0597 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; @@ -461,7 +461,7 @@ private void chooseSchema(SupportedTypes supportedTypes, int maxComplexDepth) { } objectInspectorList.add(objectInspector); - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { } typeNames.add(decoratedTypeName); } @@ -658,7 +658,7 @@ public Object randomWritable(TypeInfo typeInfo, ObjectInspector objectInspector, elementObjectInspector); boolean isStringFamily = false; PrimitiveCategory primitiveCategory = null; - if (elementTypeInfo.getCategory() == Category.PRIMITIVE) { + if (elementTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { primitiveCategory = ((PrimitiveTypeInfo) elementTypeInfo).getPrimitiveCategory(); if (primitiveCategory == PrimitiveCategory.STRING || primitiveCategory == PrimitiveCategory.BINARY || diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java index 10ed6d7ee2098b71d62f1d261fd85edfeaef9d24..6cfda8a243de4e13e27b59fdb926f4e2585b659b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; @@ -176,7 +175,7 @@ public static void verifyHashMapRowsMore(List rows, int[] actualToValu private static void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VerifyFastRow.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject); } else { final Object complexFieldObj = diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 92f005d1dc837ea5ba7d8579892b6e7325940120..ca1d60b6affeb3cbb82e93ab4032d36a6f31f773 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -97,6 +97,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.shims.CombineHiveKey; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IntWritable; @@ -366,7 +368,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java index ef678a8eb323149ee11cbe77c5b55a7c651cf474..61bfbddbe0bee448a9cf7fe10e63203b933f9c35 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java @@ -25,7 +25,6 @@ import java.io.File; import java.io.IOException; -import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.sql.Date; @@ -70,6 +69,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -314,7 +314,7 @@ public void testReadFormat_0_11() throws Exception { // check the inspectors StructObjectInspector readerInspector = (StructObjectInspector) reader .getObjectInspector(); - assertEquals(ObjectInspector.Category.STRUCT, readerInspector.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, readerInspector.getCategory()); assertEquals("struct", readerInspector.getTypeName()); @@ -943,7 +943,7 @@ public void test1() throws Exception { // check the inspectors StructObjectInspector readerInspector = (StructObjectInspector) reader.getObjectInspector(); - assertEquals(ObjectInspector.Category.STRUCT, + assertEquals(TypeInfo.Category.STRUCT, readerInspector.getCategory()); assertEquals("struct", readerInspector.getTypeName()); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java index c96a75d0079dae8baa41fd2e42efba1c7870fe8b..638e94c8d3115bedfdab7533e6eb958b27e5b59b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcStruct.java @@ -19,7 +19,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -27,7 +26,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +90,7 @@ public void testInspectorFromTypeInfo() throws Exception { assertEquals(list, inspector.getStructFieldsDataAsList(s1)); ListObjectInspector listOI = (ListObjectInspector) inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector(); - assertEquals(ObjectInspector.Category.LIST, listOI.getCategory()); + assertEquals(TypeInfo.Category.LIST, listOI.getCategory()); assertEquals(10, listOI.getListElement(list, 10)); assertEquals(null, listOI.getListElement(list, -1)); assertEquals(null, listOI.getListElement(list, 13)); diff --git a/serde/pom.xml b/serde/pom.xml index 0247c32452180aad73eb1932110096c1d044bd15..b39c6b7c91d66f83ee75a4f85081bdab679fd91c 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -49,6 +49,11 @@ hive-shims ${project.version} + + org.apache.hive + hive-standalone-metastore + ${project.version} + com.google.code.findbugs diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java index bc7d7e01422e78d44521432eae0acdbd8a7a8a76..afff081fe2b8d72791b5fff77c081b6464a301b1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/BaseStructObjectInspector.java @@ -21,6 +21,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import java.util.ArrayList; import java.util.List; @@ -125,7 +127,7 @@ public String getTypeName() { @Override public final Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java index 76c20b43b3509a3b054c5f5f3280c45e88a0231e..9285f7c03e326fea25db79836cf8d674aafc7350 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java @@ -20,13 +20,13 @@ import java.io.IOException; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.io.Writable; @@ -55,7 +55,7 @@ public Object doDeserialize(Writable field) throws SerDeException { @Override protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector, LazySerDeParameters serdeParams) throws SerDeException { - if (!objInspector.getCategory().equals(Category.PRIMITIVE) || (objInspector.getTypeName().equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME))) { + if (!objInspector.getCategory().equals(TypeInfo.Category.PRIMITIVE) || (objInspector.getTypeName().equalsIgnoreCase(serdeConstants.BINARY_TYPE_NAME))) { //do this for all complex types and binary try { serialize(out, SerDeUtils.getJSONString(obj, objInspector, serdeParams.getNullSequence().toString()), diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java index 8c375229d44dbf3353b53b78cd68ba77bc7530eb..9dd90c0228266c6f65d66763d540cc37aaca2468 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java @@ -24,13 +24,13 @@ import java.util.List; import java.util.Properties; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.MetadataListStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -203,7 +203,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -217,7 +217,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe sb.append(separator); } Object column = soi.getStructFieldData(obj, fields.get(i)); - if (fields.get(i).getFieldObjectInspector().getCategory() == Category.PRIMITIVE) { + if (fields.get(i).getFieldObjectInspector().getCategory() == TypeInfo.Category.PRIMITIVE) { // For primitive object, serialize to plain string sb.append(column == null ? nullString : column.toString()); } else { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java index db0887894431e0f316ed89d2bfe2d7b26973ae84..603a9cdc4e285b4d7db1ef0734da65fd47cbfae1 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/NullStructSerDe.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; @@ -97,7 +99,7 @@ public String getTypeName() { } public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java index 085835f8e57788122801592a99ac42eb68211342..766abad0f47b4edb3703ec23602ef61633fe4cc2 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java @@ -24,7 +24,6 @@ import java.util.Properties; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -50,6 +49,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.slf4j.Logger; @@ -166,7 +166,7 @@ public static String lightEscapeString(String str) { * This method is kept consistent with HiveResultSetMetaData#hiveTypeToSqlType . */ public static Object toThriftPayload(Object val, ObjectInspector valOI, int version) { - if (valOI.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (valOI.getCategory() == TypeInfo.Category.PRIMITIVE) { if (val == null) { return null; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java index a07cf09d11dc55262cdbac7ad45862d3cf1783a1..dd8c100cb4c06fd84987b0db1abf76a70beb5ebc 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroObjectInspectorGenerator.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; /** @@ -50,7 +51,7 @@ public AvroObjectInspectorGenerator(Schema schema) throws SerDeException { verifySchemaIsARecord(schema); this.columnNames = AvroObjectInspectorGenerator.generateColumnNames(schema); - this.columnTypes = SchemaToTypeInfo.generateColumnTypes(schema); + this.columnTypes = SchemaToHiveTypeInfo.getInstance().generateColumnTypes(schema); this.columnComments = AvroObjectInspectorGenerator.generateColumnComments(schema); assert columnNames.size() == columnTypes.size(); this.oi = createObjectInspector(); @@ -139,12 +140,12 @@ private ObjectInspector createObjectInspectorWorker(TypeInfo ti) throws SerDeExc } private boolean supportedCategories(TypeInfo ti) { - final ObjectInspector.Category c = ti.getCategory(); - return c.equals(ObjectInspector.Category.PRIMITIVE) || - c.equals(ObjectInspector.Category.MAP) || - c.equals(ObjectInspector.Category.LIST) || - c.equals(ObjectInspector.Category.STRUCT) || - c.equals(ObjectInspector.Category.UNION); + final Category c = ti.getCategory(); + return c.equals(TypeInfo.Category.PRIMITIVE) || + c.equals(TypeInfo.Category.MAP) || + c.equals(TypeInfo.Category.LIST) || + c.equals(TypeInfo.Category.STRUCT) || + c.equals(TypeInfo.Category.UNION); } public static List generateColumnNames(Schema schema) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java deleted file mode 100644 index 2d5202091505990f46586b67f0243193db2a2253..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Cache for objects whose creation only depends on some other set of objects and therefore can be - * used against other equivalent versions of those objects. Essentially memoizes instance creation. - * - * @param Object that determines the instance. The cache uses this object as a key for - * its hash which is why it is imperative to have appropriate equals and hashcode - * implementation for this object for the cache to work properly - * @param Instance that will be created from SeedObject. - */ -public abstract class InstanceCache { - private static final Logger LOG = LoggerFactory.getLogger(InstanceCache.class); - Map cache = new HashMap(); - - public InstanceCache() {} - - /** - * Retrieve (or create if it doesn't exist) the correct Instance for this - * SeedObject - */ - public Instance retrieve(SeedObject hv) throws AvroSerdeException { - return retrieve(hv, null); - } - - /** - * Retrieve (or create if it doesn't exist) the correct Instance for this - * SeedObject using 'seenSchemas' to resolve circular references - */ - public synchronized Instance retrieve(SeedObject hv, - Set seenSchemas) throws AvroSerdeException { - if(LOG.isDebugEnabled()) LOG.debug("Checking for hv: " + hv.toString()); - - if(cache.containsKey(hv)) { - if(LOG.isDebugEnabled()) LOG.debug("Returning cache result."); - return cache.get(hv); - } - - if(LOG.isDebugEnabled()) LOG.debug("Creating new instance and storing in cache"); - - Instance instance = makeInstance(hv, seenSchemas); - cache.put(hv, instance); - return instance; - } - - protected abstract Instance makeInstance(SeedObject hv, - Set seenSchemas) throws AvroSerdeException; -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java deleted file mode 100644 index 65f104dcaa37f9fff17ad1a8e690d17041cf7f06..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import org.apache.avro.Schema; - -class SchemaResolutionProblem { - static final String sentinelString = "{\n" + - " \"namespace\": \"org.apache.hadoop.hive\",\n" + - " \"name\": \"CannotDetermineSchemaSentinel\",\n" + - " \"type\": \"record\",\n" + - " \"fields\": [\n" + - " {\n" + - " \"name\":\"ERROR_ERROR_ERROR_ERROR_ERROR_ERROR_ERROR\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"Cannot_determine_schema\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"check\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"schema\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"url\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"and\",\n" + - " \"type\":\"string\"\n" + - " },\n" + - " {\n" + - " \"name\":\"literal\",\n" + - " \"type\":\"string\"\n" + - " }\n" + - " ]\n" + - "}"; - public final static Schema SIGNAL_BAD_SCHEMA = AvroSerdeUtils.getSchemaFor(sentinelString); -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..d82f5112ba9396f6a3370574a11adb1c908fbb26 --- /dev/null +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToHiveTypeInfo.java @@ -0,0 +1,18 @@ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; + +public class SchemaToHiveTypeInfo extends SchemaToTypeInfo { + private static final SchemaToHiveTypeInfo instance = new SchemaToHiveTypeInfo(); + + private SchemaToHiveTypeInfo() { + //use getInstance to get this object. The base class uses cache to reuse + //Types when available + super(TypeInfoFactory.getInstance()); + } + + public static final SchemaToHiveTypeInfo getInstance() { + return instance; + } +} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java deleted file mode 100644 index 35d83bdb1af03df674842b72422e4b7d4d22b596..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import static org.apache.avro.Schema.Type.BOOLEAN; -import static org.apache.avro.Schema.Type.BYTES; -import static org.apache.avro.Schema.Type.DOUBLE; -import static org.apache.avro.Schema.Type.FIXED; -import static org.apache.avro.Schema.Type.FLOAT; -import static org.apache.avro.Schema.Type.INT; -import static org.apache.avro.Schema.Type.LONG; -import static org.apache.avro.Schema.Type.NULL; -import static org.apache.avro.Schema.Type.STRING; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Hashtable; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.avro.Schema; -import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; - -/** - * Convert an Avro Schema to a Hive TypeInfo - */ -class SchemaToTypeInfo { - // Conversion of Avro primitive types to Hive primitive types - // Avro Hive - // Null - // boolean boolean check - // int int check - // long bigint check - // float double check - // double double check - // bytes binary check - // fixed binary check - // string string check - // tinyint - // smallint - - // Map of Avro's primitive types to Hives (for those that are supported by both) - private static final Map primitiveTypeToTypeInfo = initTypeMap(); - private static Map initTypeMap() { - Map theMap = new Hashtable(); - theMap.put(NULL, TypeInfoFactory.getPrimitiveTypeInfo("void")); - theMap.put(BOOLEAN, TypeInfoFactory.getPrimitiveTypeInfo("boolean")); - theMap.put(INT, TypeInfoFactory.getPrimitiveTypeInfo("int")); - theMap.put(LONG, TypeInfoFactory.getPrimitiveTypeInfo("bigint")); - theMap.put(FLOAT, TypeInfoFactory.getPrimitiveTypeInfo("float")); - theMap.put(DOUBLE, TypeInfoFactory.getPrimitiveTypeInfo("double")); - theMap.put(BYTES, TypeInfoFactory.getPrimitiveTypeInfo("binary")); - theMap.put(FIXED, TypeInfoFactory.getPrimitiveTypeInfo("binary")); - theMap.put(STRING, TypeInfoFactory.getPrimitiveTypeInfo("string")); - return Collections.unmodifiableMap(theMap); - } - - /** - * Generate a list of of TypeInfos from an Avro schema. This method is - * currently public due to some weirdness in deserializing unions, but - * will be made private once that is resolved. - * @param schema Schema to generate field types for - * @return List of TypeInfos, each element of which is a TypeInfo derived - * from the schema. - * @throws AvroSerdeException for problems during conversion. - */ - public static List generateColumnTypes(Schema schema) throws AvroSerdeException { - return generateColumnTypes (schema, null); - } - - /** - * Generate a list of of TypeInfos from an Avro schema. This method is - * currently public due to some weirdness in deserializing unions, but - * will be made private once that is resolved. - * @param schema Schema to generate field types for - * @param seenSchemas stores schemas processed in the parsing done so far, - * helping to resolve circular references in the schema - * @return List of TypeInfos, each element of which is a TypeInfo derived - * from the schema. - * @throws AvroSerdeException for problems during conversion. - */ - public static List generateColumnTypes(Schema schema, - Set seenSchemas) throws AvroSerdeException { - List fields = schema.getFields(); - - List types = new ArrayList(fields.size()); - - for (Schema.Field field : fields) { - types.add(generateTypeInfo(field.schema(), seenSchemas)); - } - - return types; - } - - static InstanceCache typeInfoCache = new InstanceCache() { - @Override - protected TypeInfo makeInstance(Schema s, - Set seenSchemas) - throws AvroSerdeException { - return generateTypeInfoWorker(s, seenSchemas); - } - }; - /** - * Convert an Avro Schema into an equivalent Hive TypeInfo. - * @param schema to record. Must be of record type. - * @param seenSchemas stores schemas processed in the parsing done so far, - * helping to resolve circular references in the schema - * @return TypeInfo matching the Avro schema - * @throws AvroSerdeException for any problems during conversion. - */ - public static TypeInfo generateTypeInfo(Schema schema, - Set seenSchemas) throws AvroSerdeException { - // For bytes type, it can be mapped to decimal. - Schema.Type type = schema.getType(); - if (type == BYTES && AvroSerDe.DECIMAL_TYPE_NAME - .equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int precision = 0; - int scale = 0; - try { - precision = schema.getJsonProp(AvroSerDe.AVRO_PROP_PRECISION).getIntValue(); - scale = schema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex); - } - - try { - HiveDecimalUtils.validateParameter(precision, scale); - } catch (Exception ex) { - throw new AvroSerdeException("Invalid precision or scale for decimal type", ex); - } - - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); - } - - if (type == STRING && - AvroSerDe.CHAR_TYPE_NAME.equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int maxLength = 0; - try { - maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); - } - return TypeInfoFactory.getCharTypeInfo(maxLength); - } - - if (type == STRING && AvroSerDe.VARCHAR_TYPE_NAME - .equalsIgnoreCase(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - int maxLength = 0; - try { - maxLength = schema.getJsonProp(AvroSerDe.AVRO_PROP_MAX_LENGTH).getValueAsInt(); - } catch (Exception ex) { - throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); - } - return TypeInfoFactory.getVarcharTypeInfo(maxLength); - } - - if (type == INT && - AvroSerDe.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - return TypeInfoFactory.dateTypeInfo; - } - - if (type == LONG && - AvroSerDe.TIMESTAMP_TYPE_NAME.equals(schema.getProp(AvroSerDe.AVRO_PROP_LOGICAL_TYPE))) { - return TypeInfoFactory.timestampTypeInfo; - } - - return typeInfoCache.retrieve(schema, seenSchemas); - } - - private static TypeInfo generateTypeInfoWorker(Schema schema, - Set seenSchemas) throws AvroSerdeException { - // Avro requires NULLable types to be defined as unions of some type T - // and NULL. This is annoying and we're going to hide it from the user. - if(AvroSerdeUtils.isNullableType(schema)) { - return generateTypeInfo( - AvroSerdeUtils.getOtherTypeFromNullableType(schema), seenSchemas); - } - - Schema.Type type = schema.getType(); - if(primitiveTypeToTypeInfo.containsKey(type)) { - return primitiveTypeToTypeInfo.get(type); - } - - switch(type) { - case RECORD: return generateRecordTypeInfo(schema, seenSchemas); - case MAP: return generateMapTypeInfo(schema, seenSchemas); - case ARRAY: return generateArrayTypeInfo(schema, seenSchemas); - case UNION: return generateUnionTypeInfo(schema, seenSchemas); - case ENUM: return generateEnumTypeInfo(schema); - default: throw new AvroSerdeException("Do not yet support: " + schema); - } - } - - private static TypeInfo generateRecordTypeInfo(Schema schema, - Set seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.RECORD); - - if (seenSchemas == null) { - seenSchemas = Collections.newSetFromMap(new IdentityHashMap()); - } else if (seenSchemas.contains(schema)) { - throw new AvroSerdeException( - "Recursive schemas are not supported. Recursive schema was " + schema - .getFullName()); - } - seenSchemas.add(schema); - - List fields = schema.getFields(); - List fieldNames = new ArrayList(fields.size()); - List typeInfos = new ArrayList(fields.size()); - - for(int i = 0; i < fields.size(); i++) { - fieldNames.add(i, fields.get(i).name()); - typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); - } - - return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); - } - - /** - * Generate a TypeInfo for an Avro Map. This is made slightly simpler in that - * Avro only allows maps with strings for keys. - */ - private static TypeInfo generateMapTypeInfo(Schema schema, - Set seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.MAP); - Schema valueType = schema.getValueType(); - TypeInfo ti = generateTypeInfo(valueType, seenSchemas); - - return TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.getPrimitiveTypeInfo("string"), ti); - } - - private static TypeInfo generateArrayTypeInfo(Schema schema, - Set seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.ARRAY); - Schema itemsType = schema.getElementType(); - TypeInfo itemsTypeInfo = generateTypeInfo(itemsType, seenSchemas); - - return TypeInfoFactory.getListTypeInfo(itemsTypeInfo); - } - - private static TypeInfo generateUnionTypeInfo(Schema schema, - Set seenSchemas) throws AvroSerdeException { - assert schema.getType().equals(Schema.Type.UNION); - List types = schema.getTypes(); - - - List typeInfos = new ArrayList(types.size()); - - for(Schema type : types) { - typeInfos.add(generateTypeInfo(type, seenSchemas)); - } - - return TypeInfoFactory.getUnionTypeInfo(typeInfos); - } - - // Hive doesn't have an Enum type, so we're going to treat them as Strings. - // During the deserialize/serialize stage we'll check for enumness and - // convert as such. - private static TypeInfo generateEnumTypeInfo(Schema schema) { - assert schema.getType().equals(Schema.Type.ENUM); - - return TypeInfoFactory.getPrimitiveTypeInfo("string"); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java deleted file mode 100644 index 4f8b05f5aeac3099128453c74bdef0f98986888d..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import org.apache.avro.Schema; -import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.node.JsonNodeFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * Convert Hive TypeInfo to an Avro Schema - */ -public class TypeInfoToSchema { - - private long recordCounter = 0; - - /** - * Converts Hive schema to avro schema - * - * @param columnNames Names of the hive columns - * @param columnTypes Hive Column types - * @param namespace Namespace of Avro schema - * @param name Avro schema name - * @param doc Avro schema doc - * @return Avro Schema - */ - public Schema convert(List columnNames, List columnTypes, - List columnComments, String namespace, String name, String doc) { - - List fields = new ArrayList(); - for (int i = 0; i < columnNames.size(); ++i) { - final String comment = columnComments.size() > i ? columnComments.get(i) : null; - final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), - comment); - fields.addAll(getFields(avroField)); - } - - if (name == null || name.isEmpty()) { - name = "baseRecord"; - } - - Schema avroSchema = Schema.createRecord(name, doc, namespace, false); - avroSchema.setFields(fields); - return avroSchema; - } - - private Schema.Field createAvroField(String name, TypeInfo typeInfo, String comment) { - return new Schema.Field(name, createAvroSchema(typeInfo), comment, null); - } - - private Schema createAvroSchema(TypeInfo typeInfo) { - Schema schema = null; - switch (typeInfo.getCategory()) { - case PRIMITIVE: - schema = createAvroPrimitive(typeInfo); - break; - case LIST: - schema = createAvroArray(typeInfo); - break; - case MAP: - schema = createAvroMap(typeInfo); - break; - case STRUCT: - schema = createAvroRecord(typeInfo); - break; - case UNION: - schema = createAvroUnion(typeInfo); - break; - } - - return wrapInUnionWithNull(schema); - } - - private Schema createAvroPrimitive(TypeInfo typeInfo) { - PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; - Schema schema; - switch (primitiveTypeInfo.getPrimitiveCategory()) { - case STRING: - schema = Schema.create(Schema.Type.STRING); - break; - case CHAR: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.CHAR_TYPE_NAME + "\"," + - "\"maxLength\":" + ((CharTypeInfo) typeInfo).getLength() + "}"); - break; - case VARCHAR: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_STRING_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.VARCHAR_TYPE_NAME + "\"," + - "\"maxLength\":" + ((VarcharTypeInfo) typeInfo).getLength() + "}"); - break; - case BINARY: - schema = Schema.create(Schema.Type.BYTES); - break; - case BYTE: - schema = Schema.create(Schema.Type.INT); - break; - case SHORT: - schema = Schema.create(Schema.Type.INT); - break; - case INT: - schema = Schema.create(Schema.Type.INT); - break; - case LONG: - schema = Schema.create(Schema.Type.LONG); - break; - case FLOAT: - schema = Schema.create(Schema.Type.FLOAT); - break; - case DOUBLE: - schema = Schema.create(Schema.Type.DOUBLE); - break; - case BOOLEAN: - schema = Schema.create(Schema.Type.BOOLEAN); - break; - case DECIMAL: - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; - String precision = String.valueOf(decimalTypeInfo.precision()); - String scale = String.valueOf(decimalTypeInfo.scale()); - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"bytes\"," + - "\"logicalType\":\"decimal\"," + - "\"precision\":" + precision + "," + - "\"scale\":" + scale + "}"); - break; - case DATE: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_INT_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.DATE_TYPE_NAME + "\"}"); - break; - case TIMESTAMP: - schema = AvroSerdeUtils.getSchemaFor("{" + - "\"type\":\"" + AvroSerDe.AVRO_LONG_TYPE_NAME + "\"," + - "\"logicalType\":\"" + AvroSerDe.TIMESTAMP_TYPE_NAME + "\"}"); - break; - case VOID: - schema = Schema.create(Schema.Type.NULL); - break; - default: - throw new UnsupportedOperationException(typeInfo + " is not supported."); - } - return schema; - } - - private Schema createAvroUnion(TypeInfo typeInfo) { - List childSchemas = new ArrayList(); - for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) { - final Schema childSchema = createAvroSchema(childTypeInfo); - if (childSchema.getType() == Schema.Type.UNION) { - childSchemas.addAll(childSchema.getTypes()); - } else { - childSchemas.add(childSchema); - } - } - - return Schema.createUnion(removeDuplicateNullSchemas(childSchemas)); - } - - private Schema createAvroRecord(TypeInfo typeInfo) { - List childFields = new ArrayList(); - - final List allStructFieldNames = - ((StructTypeInfo) typeInfo).getAllStructFieldNames(); - final List allStructFieldTypeInfos = - ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); - if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { - throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + - "name and column type differs. names = " + allStructFieldNames + ", types = " + - allStructFieldTypeInfos); - } - - for (int i = 0; i < allStructFieldNames.size(); ++i) { - final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); - final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), - childTypeInfo, childTypeInfo.toString()); - final List grandChildFields = getFields(grandChildSchemaField); - childFields.addAll(grandChildFields); - } - - Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), - null, false); - ++recordCounter; - recordSchema.setFields(childFields); - return recordSchema; - } - - private Schema createAvroMap(TypeInfo typeInfo) { - TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); - if (((PrimitiveTypeInfo) keyTypeInfo).getPrimitiveCategory() - != PrimitiveObjectInspector.PrimitiveCategory.STRING) { - throw new UnsupportedOperationException("Key of Map can only be a String"); - } - - TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); - Schema valueSchema = createAvroSchema(valueTypeInfo); - - return Schema.createMap(valueSchema); - } - - private Schema createAvroArray(TypeInfo typeInfo) { - ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; - Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo()); - return Schema.createArray(listSchema); - } - - private List getFields(Schema.Field schemaField) { - List fields = new ArrayList(); - - JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); - if (schemaField.schema().getType() == Schema.Type.RECORD) { - for (Schema.Field field : schemaField.schema().getFields()) { - fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); - } - } else { - fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), - nullDefault)); - } - - return fields; - } - - private Schema wrapInUnionWithNull(Schema schema) { - Schema wrappedSchema = schema; - switch (schema.getType()) { - case NULL: - break; - case UNION: - List existingSchemas = removeDuplicateNullSchemas(schema.getTypes()); - wrappedSchema = Schema.createUnion(existingSchemas); - break; - default: - wrappedSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema)); - } - - return wrappedSchema; - } - - private List removeDuplicateNullSchemas(List childSchemas) { - List prunedSchemas = new ArrayList(); - boolean isNullPresent = false; - for (Schema schema : childSchemas) { - if (schema.getType() == Schema.Type.NULL) { - isNullPresent = true; - } else { - prunedSchemas.add(schema); - } - } - if (isNullPresent) { - prunedSchemas.add(0, Schema.create(Schema.Type.NULL)); - } - - return prunedSchemas; - } -} \ No newline at end of file diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java index 461043d5873b3e07db8b8c28e24113e94e296455..a2ca42bc58b1cbcd53b3812fd14370947a885f4b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java @@ -24,7 +24,7 @@ import java.util.Deque; import java.util.List; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -107,7 +107,7 @@ public BinarySortableDeserializeRead(TypeInfo[] typeInfos, boolean useExternalBu final int count = typeInfos.length; root = new Field(); - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = count; stack = new ArrayDeque<>(); @@ -502,7 +502,7 @@ public void skipNextField() throws IOException { return; } - if (current.category == Category.UNION && current.index == 0) { + if (current.category == TypeInfo.Category.UNION && current.index == 0) { current.tag = inputByteBuffer.read(); currentInt = current.tag; return; @@ -513,7 +513,7 @@ public void skipNextField() throws IOException { if (isNull()) { return; } - if (child.category == Category.PRIMITIVE) { + if (child.category == TypeInfo.Category.PRIMITIVE) { readPrimitive(child); } else { stack.push(child); @@ -659,7 +659,7 @@ public boolean readComplexField() throws IOException { return false; } - if (current.category == Category.UNION) { + if (current.category == TypeInfo.Category.UNION) { if (current.index == 0) { current.tag = inputByteBuffer.read(columnSortOrderIsDesc[root.index]); currentInt = current.tag; @@ -674,7 +674,7 @@ public boolean readComplexField() throws IOException { if (isNull) { return false; } - if (child.category == Category.PRIMITIVE) { + if (child.category == TypeInfo.Category.PRIMITIVE) { isNull = !readPrimitive(child); } else { stack.push(child); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java index 2b10c2764475c7e065b1906f83f8ecda8a4411de..8a126bc78a6915d046320756a22c2afbb37322a6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Properties; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -36,7 +37,6 @@ import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; @@ -128,7 +128,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -167,10 +167,10 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe // the field is declared as a primitive in initialization, serialize // the data to JSON string. Otherwise serialize the data in the // delimited way. - if (!foi.getCategory().equals(Category.PRIMITIVE) + if (!foi.getCategory().equals(TypeInfo.Category.PRIMITIVE) && (declaredFields == null || declaredFields.get(i) .getFieldObjectInspector().getCategory().equals( - Category.PRIMITIVE))) { + TypeInfo.Category.PRIMITIVE))) { LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString( f, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java index 44953557b87f67cb96a3d251bd1194e6c7abfd4d..384b8830657f6b2ab5a0e6bdfdf417bba9c93254 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -86,7 +85,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException static final byte[] INVALID_UTF__SINGLE_BYTE = {(byte)Integer.parseInt("10111111", 2)}; @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -107,7 +106,7 @@ public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDe Object f = (list == null ? null : list.get(i)); //empty strings are marked by an invalid utf single byte sequence. A valid utf stream cannot //produce this sequence - if ((f != null) && (foi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) + if ((f != null) && (foi.getCategory().equals(TypeInfo.Category.PRIMITIVE)) && ((PrimitiveObjectInspector) foi).getPrimitiveCategory().equals( PrimitiveObjectInspector.PrimitiveCategory.STRING) && ((StringObjectInspector) foi).getPrimitiveJavaObject(f).length() == 0) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java index d3e32483377b74178bebb071b71011aeda1c9816..24e41c7cf9819b4a8b27d525db6338afde94fb09 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java @@ -24,9 +24,10 @@ import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; public class LazyBinaryColumnarStruct extends ColumnarStructBase { @@ -41,7 +42,7 @@ protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByte return -1; } Category category = objectInspector.getCategory(); - if (category.equals(Category.PRIMITIVE)) { + if (category.equals(TypeInfo.Category.PRIMITIVE)) { PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector) .getPrimitiveCategory(); if (primitiveCategory.equals(PrimitiveCategory.STRING) && (length == 1) && diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java index c7f49305d5e9b73594997166b9e10031a8dbfb89..532ac05a00e32c8a53f66fc8ad346e9a21086d78 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.thrift.TException; import org.apache.thrift.protocol.TMap; import org.apache.thrift.protocol.TProtocol; @@ -129,7 +130,7 @@ public void serialize(Object o, ObjectInspector oi, TProtocol oprot) WriteNullsProtocol nullProtocol = (oprot instanceof WriteNullsProtocol) ? (WriteNullsProtocol) oprot : null; - assert (oi.getCategory() == ObjectInspector.Category.MAP); + assert (oi.getCategory() == TypeInfo.Category.MAP); MapObjectInspector moi = (MapObjectInspector) oi; ObjectInspector koi = moi.getMapKeyObjectInspector(); ObjectInspector voi = moi.getMapValueObjectInspector(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java index 197031de1109f446fd1cd3e5f74841d44915129c..e78fe81861d3160d75c8a5d3dc1fc751f07e7e7d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/fast/DeserializeRead.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; @@ -163,7 +163,7 @@ public DeserializeRead(TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTyp TypeInfo typeInfo = typeInfos[i]; Category category = typeInfo.getCategory(); categories[i] = category; - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo; PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory(); primitiveCategories[i] = primitiveCategory; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java index 883ba88df624761bb59b5ea9edebd2b8157625a9..175cb39d08327cc5b21a7306344b09ccb5b1fe85 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java @@ -58,7 +58,7 @@ import org.apache.hadoop.hive.serde2.lazydio.LazyDioLong; import org.apache.hadoop.hive.serde2.lazydio.LazyDioShort; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -180,7 +180,7 @@ * Create a hierarchical LazyObject based on the given typeInfo. */ public static LazyObject createLazyObject(ObjectInspector oi) { - ObjectInspector.Category c = oi.getCategory(); + Category c = oi.getCategory(); switch (c) { case PRIMITIVE: return createLazyPrimitiveClass((PrimitiveObjectInspector) oi); @@ -209,7 +209,7 @@ public static LazyObject createLazyObject(ObjectInspector oi, boolean typeBinary) { - if (oi.getCategory() == Category.PRIMITIVE) { + if (oi.getCategory() == TypeInfo.Category.PRIMITIVE) { return createLazyPrimitiveClass((PrimitiveObjectInspector) oi, typeBinary); } else { return createLazyObject(oi); @@ -325,7 +325,7 @@ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, int separatorIndex, LazyObjectInspectorParameters lazyParams, ObjectInspectorOptions option) throws SerDeException { - ObjectInspector.Category c = typeInfo.getCategory(); + Category c = typeInfo.getCategory(); switch (c) { case PRIMITIVE: return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector( diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java index fb194c8a74869114fd5531d1767b831ddd6b5e63..5ca292e86408ebcaf62e00e3fcc39c17b4a5f72d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java @@ -38,13 +38,13 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.BinaryComparable; import org.apache.hadoop.io.Text; @@ -192,7 +192,7 @@ public ObjectInspector getObjectInspector() throws SerDeException { public Writable doSerialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index fe0ee48b4f3bb5a04f2b4f03479b5d9593706984..697ad069f60fa61699f97939521646d0a5c0acdf 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters; import org.apache.hadoop.hive.serde2.lazy.LazyShort; import org.apache.hadoop.hive.serde2.lazy.LazyUtils; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -90,7 +90,7 @@ public Field(TypeInfo typeInfo, DataTypePhysicalVariation dataTypePhysicalVariation) { Category category = typeInfo.getCategory(); - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { isPrimitive = true; primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); complexCategory = null; @@ -882,7 +882,7 @@ private boolean doReadField(Field field) { } ComplexTypeHelper complexTypeHelper = field.complexTypeHelper; currentComplexTypeHelpers[currentLevel++] = complexTypeHelper; - if (field.complexCategory == Category.MAP) { + if (field.complexCategory == TypeInfo.Category.MAP) { currentComplexTypeHelpers[currentLevel] = null; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java index 3c13df4b698e945b2470a5ac67073eaa03a1fc5e..1682a0681bd0f6a784fc658baf302a47115343ef 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java @@ -20,11 +20,12 @@ import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde2.lazy.LazyArray; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.io.Text; @@ -59,7 +60,7 @@ protected LazyListObjectInspector(ObjectInspector listElementObjectInspector, @Override public final Category getCategory() { - return Category.LIST; + return TypeInfo.Category.LIST; } // without data diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java index e55089d062e5baa77a7bc6ac13a52ea4f7051151..b092b765e0dba348f08fef58c1a6e3e6eebde921 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java @@ -20,6 +20,8 @@ import java.util.Map; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde2.lazy.LazyMap; @@ -79,7 +81,7 @@ protected LazyMapObjectInspector(ObjectInspector mapKeyObjectInspector, @Override public final Category getCategory() { - return Category.MAP; + return TypeInfo.Category.MAP; } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java index f0953bd58437987b694b51b8af99f4a0353a542d..30b76dfb2571560c71607cea2aa2426e9ff76e73 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyUnionObjectInspector.java @@ -21,11 +21,12 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde2.lazy.LazyUnion; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; @@ -74,7 +75,7 @@ protected void init( @Override public final Category getCategory() { - return Category.UNION; + return TypeInfo.Category.UNION; } public byte getSeparator() { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java index 40087ae5d412073838fcf34ed71093392df6415b..0bd2342a7138938e8337c732375ac0e4f6881dae 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampLocalTZObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; /** * LazyBinaryFactory. @@ -106,7 +107,7 @@ * Create a hierarchical LazyBinaryObject based on the given typeInfo. */ public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi) { - ObjectInspector.Category c = oi.getCategory(); + Category c = oi.getCategory(); switch (c) { case PRIMITIVE: return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index 24704a1935d28162e8ad59bb6c14f4f3cf5a089c..ffab3df32bd51d799d202e59b5a61c8babd9652f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -48,7 +48,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -196,7 +195,7 @@ public Object deserialize(Writable field) throws SerDeException { public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { // make sure it is a struct record - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); @@ -669,7 +668,7 @@ public static void serialize(RandomAccessOutput byteStream, Object obj, typeStart = byteStream.getLength(); } - if (ObjectInspector.Category.STRUCT.equals(objInspector.getCategory()) ) { + if (TypeInfo.Category.STRUCT.equals(objInspector.getCategory()) ) { // 2/ serialize the struct serializeStruct(byteStream, obj, (StructObjectInspector) objInspector, warnedOnceNullMapKey); } else { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java index b3285084088939d7e227142a0cc82bccfaeecb23..dfdbae1b5134ff4c61ca596f0c1d94056e9a3b5b 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -56,6 +55,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -83,7 +83,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException @Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { - if (objInspector.getCategory() != Category.STRUCT) { + if (objInspector.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java index ee1e2e6e0fe11fd16e0dac8504e30e7aa67231be..813f454f4f4745029ff9b0c7998ad7d767b9e263 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java index 340f322c659c58ac7bcc85ad51e4bd15025f56f8..9a5c4a9764c1a59e131e45fdaf7f01a8b170086e 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VLong; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; @@ -97,7 +97,7 @@ public LazyBinaryDeserializeRead(TypeInfo[] typeInfos, boolean useExternalBuffer currentExternalBufferNeeded = false; root = new Field(); - root.category = Category.STRUCT; + root.category = TypeInfo.Category.STRUCT; root.children = createFields(typeInfos); root.count = typeInfos.length; } @@ -422,7 +422,7 @@ public void skipNextField() throws IOException { final Field child = getChild(current); - if (child.category == Category.PRIMITIVE) { + if (child.category == TypeInfo.Category.PRIMITIVE) { readPrimitive(child); current.index++; } else { @@ -493,7 +493,7 @@ private void parseHeader(Field field) { case MAP: // Read count LazyBinaryUtils.readVInt(bytes, offset, tempVInt); - if (field.category == Category.LIST) { + if (field.category == TypeInfo.Category.LIST) { field.count = tempVInt.value; } else { field.count = tempVInt.value * 2; @@ -529,7 +529,7 @@ private Field getChild(Field field) { } private boolean readUnionTag(Field field) { - if (field.category == Category.UNION && field.index == 0) { + if (field.category == TypeInfo.Category.UNION && field.index == 0) { field.tag = bytes[offset++]; currentInt = field.tag; return true; @@ -556,7 +556,7 @@ public boolean readComplexField() throws IOException { final Field child = getChild(current); - if (child.category == Category.PRIMITIVE) { + if (child.category == TypeInfo.Category.PRIMITIVE) { isNull = !readPrimitive(child); current.index++; } else { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java index cd4e619e3b787a16cd5bd183e0fab13e58a5cadc..259bc16dcb6faa222b93e94b7b1d9158db7987f4 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java @@ -27,7 +27,7 @@ import java.util.Map; import org.apache.hadoop.hive.serde2.ByteStream; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.common.type.HiveChar; @@ -45,10 +45,10 @@ import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils; import org.apache.hadoop.hive.serde2.fast.SerializeWrite; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.LIST; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.MAP; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.STRUCT; -import static org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.UNION; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category.LIST; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category.MAP; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category.STRUCT; +import static org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category.UNION; /* * Directly serialize, field-by-field, the LazyBinary format. diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java index 99b565dd3fcb14fa1b05408e48a776260d1b47ac..07171b18ec2c23dbe3779fcfbfc149cd23cde7d6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; /** * ObjectInspector helps us to look into the internal structure of a complex @@ -41,15 +42,6 @@ @InterfaceAudience.Public @InterfaceStability.Stable public interface ObjectInspector extends Cloneable { - - /** - * Category. - * - */ - public static enum Category { - PRIMITIVE, LIST, MAP, STRUCT, UNION - }; - /** * Returns the name of the data type that is inspected by this * ObjectInspector. This is used to display the type information to the user. diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index efb1df6495dff1ec7a1498b7ef31ae631378c2a4..785368fccb090be91d4bfe89098cd774cdcaf410 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampLocalTZObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde.serdeConstants; @@ -42,7 +43,7 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazy.LazyDouble; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveWritableObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; @@ -186,7 +187,7 @@ public static int writableArrayHashCode(Object[] keys) { */ public static ObjectInspector getWritableObjectInspector(ObjectInspector oi) { // All non-primitive OIs are writable so we need only check this case. - if (oi.getCategory() == Category.PRIMITIVE) { + if (oi.getCategory() == TypeInfo.Category.PRIMITIVE) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; if (!(poi instanceof AbstractPrimitiveWritableObjectInspector)) { return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector( @@ -370,7 +371,7 @@ public static Object copyToStandardJavaObject(Object o, ObjectInspector oi) { } public static int getStructSize(ObjectInspector oi) throws SerDeException { - if (oi.getCategory() != Category.STRUCT) { + if (oi.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException("Unexpected category " + oi.getCategory()); } return ((StructObjectInspector)oi).getAllStructFieldRefs().size(); @@ -383,7 +384,7 @@ public static void copyStructToArray(Object o, ObjectInspector oi, return; } - if (oi.getCategory() != Category.STRUCT) { + if (oi.getCategory() != TypeInfo.Category.STRUCT) { throw new SerDeException("Unexpected category " + oi.getCategory()); } @@ -1134,12 +1135,12 @@ public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) { } // If both categories are primitive return the comparison of type names. - if (c1.equals(Category.PRIMITIVE)) { + if (c1.equals(TypeInfo.Category.PRIMITIVE)) { return o1.getTypeName().equals(o2.getTypeName()); } // If lists, recursively compare the list element types - if (c1.equals(Category.LIST)) { + if (c1.equals(TypeInfo.Category.LIST)) { ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector(); ObjectInspector child2 = @@ -1148,7 +1149,7 @@ public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) { } // If maps, recursively compare the key and value types - if (c1.equals(Category.MAP)) { + if (c1.equals(TypeInfo.Category.MAP)) { MapObjectInspector mapOI1 = (MapObjectInspector) o1; MapObjectInspector mapOI2 = (MapObjectInspector) o2; @@ -1168,7 +1169,7 @@ public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) { } // If structs, recursively compare the fields - if (c1.equals(Category.STRUCT)) { + if (c1.equals(TypeInfo.Category.STRUCT)) { StructObjectInspector structOI1 = (StructObjectInspector) o1; StructObjectInspector structOI2 = (StructObjectInspector) o2; @@ -1200,7 +1201,7 @@ public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) { return true; } - if (c1.equals(Category.UNION)) { + if (c1.equals(TypeInfo.Category.UNION)) { UnionObjectInspector uoi1 = (UnionObjectInspector) o1; UnionObjectInspector uoi2 = (UnionObjectInspector) o2; List ois1 = uoi1.getObjectInspectors(); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java index 3a88a400250afa0e29001ec118e006f937678089..9bb3f18b11be2e0a04fbdbb0642524980c46ef42 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ReflectionStructObjectInspector.java @@ -25,6 +25,8 @@ import java.util.Map; import java.util.Set; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.util.ReflectionUtils; /** @@ -87,7 +89,7 @@ public String toString() { volatile Type type; public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } public String getTypeName() { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java index da35c5c9015ba0f894858def49880bcea1009d8d..67dde83ca606a79bd76aad5399a2ae74d0794884 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardListObjectInspector.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; + import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -45,7 +48,7 @@ protected StandardListObjectInspector( } public final Category getCategory() { - return Category.LIST; + return TypeInfo.Category.LIST; } // without data diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java index de41b97bbd4f99fcb557d38335b86bc3ab1f054a..48768194274e1f002f46d711176f66550d7356ea 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardMapObjectInspector.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; + import java.util.HashMap; import java.util.Map; @@ -87,7 +90,7 @@ public int getMapSize(Object data) { } public final Category getCategory() { - return Category.MAP; + return TypeInfo.Category.MAP; } public String getTypeName() { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java index 3de9a2ee31d6076fadef2156e9f5312d6f623bc0..b047b15218dc1facfef4cee02580d20b245b016a 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java @@ -22,6 +22,8 @@ import java.util.Arrays; import java.util.List; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -144,7 +146,7 @@ public String getTypeName() { } public final Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } // Without Data diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java index 002514f3041b179e3393ce368fba6d6d19a4a19a..6ddff9c4a4e780c029d1e16fa151f023a6bc3d03 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; -import java.util.ArrayList; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; + import java.util.List; /** @@ -127,7 +129,7 @@ public Object getField(Object o) { } public Category getCategory() { - return Category.UNION; + return TypeInfo.Category.UNION; } public String getTypeName() { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SubStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SubStructObjectInspector.java index 8cc0efc86624fbe237d479060bf0280ca92b9ab0..cee747bb5761007865953cbd64994061361573e8 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SubStructObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SubStructObjectInspector.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; + import java.util.ArrayList; import java.util.List; @@ -59,7 +62,7 @@ public String getTypeName() { @Override public Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java index cdfaee28bbdbcd6ec64a9f77fff01f291222807c..248839362a7978abfd6cb9938a7cf54c0e2e08ff 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.thrift.TFieldIdEnum; import org.apache.thrift.TUnion; import org.apache.thrift.meta_data.FieldMetaData; @@ -112,7 +114,7 @@ protected void init(Type type, Class objectClass, @Override public Category getCategory() { - return Category.UNION; + return TypeInfo.Category.UNION; } @Override diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java index 34fd40c5cd10ebf029f8733b291ef13c2e779ef1..b9ab586ca0ecbffbc350fd848e3a947d6ba38b9d 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnionStructObjectInspector.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.serde2.objectinspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; + import java.util.ArrayList; import java.util.List; @@ -116,7 +119,7 @@ public int hashCode() { } public final Category getCategory() { - return Category.STRUCT; + return TypeInfo.Category.STRUCT; } public String getTypeName() { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java index 8bccd0f32a0b79f20d518a30d805a078949f8383..61725fafce8c6927327ca7cffce124c617f5b086 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java @@ -20,6 +20,8 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; /** * An AbstractPrimitiveObjectInspector is based on @@ -72,7 +74,7 @@ public PrimitiveCategory getPrimitiveCategory() { */ @Override public Category getCategory() { - return Category.PRIMITIVE; + return TypeInfo.Category.PRIMITIVE; } /** diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 8cf074452976aba0d98cd92c94448aa073f1a7b7..4ba14c17a8914ac4cf800054b53881af0c704876 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -55,9 +55,9 @@ import org.apache.hadoop.hive.serde2.lazy.LazyInteger; import org.apache.hadoop.hive.serde2.lazy.LazyLong; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.FloatWritable; @@ -1385,7 +1385,7 @@ public static HiveIntervalDayTime getHiveIntervalDayTime(Object o, PrimitiveObje } public static Class getJavaPrimitiveClassFromObjectInspector(ObjectInspector oi) { - if (oi.getCategory() != Category.PRIMITIVE) { + if (oi.getCategory() != TypeInfo.Category.PRIMITIVE) { return null; } PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java index 820fb4ef1dc17479d9e7fea0890d5834b806106d..4ffe50fd95dabdb33c957591c5686c142733ca53 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/BaseCharTypeInfo.java @@ -67,4 +67,9 @@ public void setTypeName(String typeName) { // type name should already be set by subclass return; } + + @Override + public Object[] getParameters() { + return new Object[] { length }; + } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java index 2e76df51e77f6e2c965deab17e04483d92149660..c5cb32b5eefd877770566de9dd716224e01a7ea4 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java @@ -127,4 +127,9 @@ public void setScale(int scale) { this.scale = scale; } + @Override + public Object[] getParameters() { + return new Object[] { precision, scale }; + } + } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java deleted file mode 100644 index c632bffeb3cf10e0c87b70342f9541773d638ece..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A List Type has homogeneous elements. All elements of the List has the same - * TypeInfo which is returned by getListElementTypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class ListTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - private TypeInfo listElementTypeInfo; - - /** - * For java serialization use only. - */ - public ListTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" - + listElementTypeInfo.getTypeName() + ">"; - } - - /** - * For java serialization use only. - */ - public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { - this.listElementTypeInfo = listElementTypeInfo; - } - - /** - * For TypeInfoFactory use only. - */ - ListTypeInfo(TypeInfo elementTypeInfo) { - listElementTypeInfo = elementTypeInfo; - } - - @Override - public Category getCategory() { - return Category.LIST; - } - - public TypeInfo getListElementTypeInfo() { - return listElementTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof ListTypeInfo)) { - return false; - } - return getListElementTypeInfo().equals( - ((ListTypeInfo) other).getListElementTypeInfo()); - } - - @Override - public int hashCode() { - return listElementTypeInfo.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java deleted file mode 100644 index 1344c902293d57e26dab6c52d8a454e9610217c0..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * A Map Type has homogeneous keys and homogeneous values. All keys of the Map - * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all - * values of the Map has the same TypeInfo, which is returned by - * getMapValueTypeInfo(). - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class MapTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private TypeInfo mapKeyTypeInfo; - private TypeInfo mapValueTypeInfo; - - /** - * For java serialization use only. - */ - public MapTypeInfo() { - } - - @Override - public String getTypeName() { - return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" - + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() - + ">"; - } - - /** - * For java serialization use only. - */ - public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { - this.mapKeyTypeInfo = mapKeyTypeInfo; - } - - /** - * For java serialization use only. - */ - public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { - this.mapValueTypeInfo = mapValueTypeInfo; - } - - // For TypeInfoFactory use only - MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { - mapKeyTypeInfo = keyTypeInfo; - mapValueTypeInfo = valueTypeInfo; - } - - @Override - public Category getCategory() { - return Category.MAP; - } - - public TypeInfo getMapKeyTypeInfo() { - return mapKeyTypeInfo; - } - - public TypeInfo getMapValueTypeInfo() { - return mapValueTypeInfo; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof MapTypeInfo)) { - return false; - } - MapTypeInfo o = (MapTypeInfo) other; - return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) - && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); - } - - @Override - public int hashCode() { - return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java index 97af49a60494991d117ccdf0ea2bfb170a827da3..0a56273f3022b5de89647dd504d3543a45d0967f 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java @@ -22,7 +22,6 @@ import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; @@ -36,31 +35,21 @@ */ @InterfaceAudience.Public @InterfaceStability.Stable -public class PrimitiveTypeInfo extends TypeInfo implements Serializable { +public class PrimitiveTypeInfo extends MetastorePrimitiveTypeInfo implements Serializable { private static final long serialVersionUID = 1L; - // Base name (varchar vs fully qualified name such as varchar(200)). - protected String typeName; - /** * For java serialization use only. */ public PrimitiveTypeInfo() { + super(); } /** * For TypeInfoFactory use only. */ PrimitiveTypeInfo(String typeName) { - this.typeName = typeName; - } - - /** - * Returns the category of this TypeInfo. - */ - @Override - public Category getCategory() { - return Category.PRIMITIVE; + super(typeName); } public PrimitiveCategory getPrimitiveCategory() { @@ -75,44 +64,7 @@ public PrimitiveCategory getPrimitiveCategory() { return getPrimitiveTypeEntry().primitiveJavaClass; } - // The following 2 methods are for java serialization use only. - public void setTypeName(String typeName) { - this.typeName = typeName; - } - - @Override - public String getTypeName() { - return typeName; - } - public PrimitiveTypeEntry getPrimitiveTypeEntry() { return PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName); } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (other == null || getClass() != other.getClass()) { - return false; - } - - PrimitiveTypeInfo pti = (PrimitiveTypeInfo) other; - - return this.typeName.equals(pti.typeName); - } - - /** - * Generate the hashCode for this TypeInfo. - */ - @Override - public int hashCode() { - return typeName.hashCode(); - } - - @Override - public String toString() { - return typeName; - } } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java deleted file mode 100644 index 4caedb02d8d765245d068482ba2cb293dad1526a..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or - * more fields each of which has a unique name and its own TypeInfo. Different - * fields can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public final class StructTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private ArrayList allStructFieldNames; - private ArrayList allStructFieldTypeInfos; - - /** - * For java serialization use only. - */ - public StructTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.STRUCT_TYPE_NAME + "<"); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allStructFieldNames.get(i)); - sb.append(":"); - sb.append(allStructFieldTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldNames(ArrayList allStructFieldNames) { - this.allStructFieldNames = allStructFieldNames; - } - - /** - * For java serialization use only. - */ - public void setAllStructFieldTypeInfos( - ArrayList allStructFieldTypeInfos) { - this.allStructFieldTypeInfos = allStructFieldTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - StructTypeInfo(List names, List typeInfos) { - allStructFieldNames = new ArrayList(names); - allStructFieldTypeInfos = new ArrayList(typeInfos); - } - - @Override - public Category getCategory() { - return Category.STRUCT; - } - - public ArrayList getAllStructFieldNames() { - return allStructFieldNames; - } - - public ArrayList getAllStructFieldTypeInfos() { - return allStructFieldTypeInfos; - } - - public TypeInfo getStructFieldTypeInfo(String field) { - String fieldLowerCase = field.toLowerCase(); - for (int i = 0; i < allStructFieldNames.size(); i++) { - if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { - return allStructFieldTypeInfos.get(i); - } - } - throw new RuntimeException("cannot find field " + field - + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); - // return null; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof StructTypeInfo)) { - return false; - } - StructTypeInfo o = (StructTypeInfo) other; - Iterator namesIterator = getAllStructFieldNames().iterator(); - Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); - - // Compare the field names using ignore-case semantics - while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { - if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { - return false; - } - } - - // Different number of field names - if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { - return false; - } - - // Compare the field types - return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); - } - - @Override - public int hashCode() { - return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java index 6f9eeea02035ef9e06ee4c5f4d8a443eed36ce6c..8b942086db3cd7ae4b0d462eeccb803fd68a7eea 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TimestampLocalTZTypeInfo.java @@ -101,4 +101,9 @@ public void setTimeZone(ZoneId timeZone) { this.timeZone = timeZone; } + @Override + public Object[] getParameters() { + return new Object[] { timeZone }; + } + } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java deleted file mode 100644 index 75e09739b9814b1428f94f71516645417981cbc7..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * Stores information about a type. Always use the TypeInfoFactory to create new - * TypeInfo objects. - * - * We support 8 categories of types: - * 1. Primitive objects (String, Number, etc) - * 2. List objects (a list of objects of a single type) - * 3. Map objects (a map from objects of one type to objects of another type) - * 4. Struct objects (a list of fields with names and their own types) - * 5. Union objects - * 6. Decimal objects - * 7. Char objects - * 8. Varchar objects - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public abstract class TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - protected TypeInfo() { - } - - /** - * The Category of this TypeInfo. Possible values are Primitive, List, Map, - * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. - */ - public abstract Category getCategory(); - - /** - * A String representation of the TypeInfo. - */ - public abstract String getTypeName(); - - /** - * String representing the qualified type name. - * Qualified types should override this method. - * @return - */ - public String getQualifiedName() { - return getTypeName(); - } - - @Override - public String toString() { - return getTypeName(); - } - - @Override - public abstract boolean equals(Object o); - - @Override - public abstract int hashCode(); - - public boolean accept(TypeInfo other) { - return this.equals(other); - } - -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java index 77d60c59c219d654d49fdd4853ce33543669c9bc..e730e9d9588b2d1c0aa4c99b9395c50f4f476d7c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser.PrimitiveParts; /** * TypeInfoFactory can be used to create the TypeInfo object for any types. @@ -39,6 +40,53 @@ */ public final class TypeInfoFactory { + //if this singleton looks weird, it is. It is done to support the move of TypeInfo and its sub-classes + //to standalone metastore. Standalone metastore needs access to TypeInfos without depending + //on hive-serde. In order to create these TypeInfo, it needs the TypeInfoParser and TypeInfoFactory. + //We cannot move TypeInfoFactory out of hive code base since its too disruptive. + //ITypeInfoFactory interface in standalone-metastore abstracts out the common functionality + //and metastore implements its own version of TypeInfoFactory. Hive uses its original TypeInfoFactory + //without any changes. The singleton weird-ness is needed to keep the caching behaviour when + //using this TypeInfoFactory in Deserializer and TypeInfoParsers + private static final ITypeInfoFactory instance = new ITypeInfoFactory() { + @Override + public MetastorePrimitiveTypeInfo getPrimitiveTypeInfo(String typeName, Object... parameters) { + if (serdeConstants.DECIMAL_TYPE_NAME.equals(typeName)) { + HiveDecimalUtils.validateParameter((Integer) parameters[0], (Integer) parameters[1]); + } else if (serdeConstants.CHAR_TYPE_NAME.equals(typeName)) { + BaseCharUtils.validateCharParameter((Integer) parameters[0]); + } else if (serdeConstants.VARCHAR_TYPE_NAME.equals(typeName)) { + BaseCharUtils.validateVarcharParameter((Integer)parameters[0]); + } + return TypeInfoFactory.getPrimitiveTypeInfo( + MetastoreTypeInfoUtils.getQualifiedPrimitiveTypeName(typeName, parameters)); + } + + @Override + public MapTypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + return (MapTypeInfo) TypeInfoFactory.getMapTypeInfo(keyTypeInfo, valueTypeInfo); + } + + @Override + public ListTypeInfo getListTypeInfo(TypeInfo listElementTypeInfo) { + return (ListTypeInfo) TypeInfoFactory.getListTypeInfo(listElementTypeInfo); + } + + @Override + public UnionTypeInfo getUnionTypeInfo(List typeInfos) { + return (UnionTypeInfo) TypeInfoFactory.getUnionTypeInfo(typeInfos); + } + + @Override + public StructTypeInfo getStructTypeInfo(List names, List typeInfos) { + return (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(names, typeInfos); + } + }; + + public static final ITypeInfoFactory getInstance() { + return instance; + } + private TypeInfoFactory() { // prevent instantiation } @@ -134,14 +182,14 @@ public static PrimitiveTypeInfo getPrimitiveTypeInfo(String typeName) { * @return PrimitiveTypeInfo instance */ private static PrimitiveTypeInfo createPrimitiveTypeInfo(String fullName) { - String baseName = TypeInfoUtils.getBaseName(fullName); + String baseName = MetastoreTypeInfoUtils.getBaseName(fullName); PrimitiveTypeEntry typeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(baseName); if (null == typeEntry) { throw new RuntimeException("Unknown type " + fullName); } - TypeInfoUtils.PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName); + PrimitiveParts parts = TypeInfoUtils.parsePrimitiveParts(fullName); if (parts.typeParams == null || parts.typeParams.length < 1) { return null; } diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java index f3b19f079173ec2f347495346708dc0ca21f70a3..a013727a92699aa06231d9f9294df94a998e6eba 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java @@ -25,18 +25,14 @@ import java.util.ArrayList; import java.util.EnumMap; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveVarchar; -import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -47,7 +43,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser.PrimitiveParts; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -240,15 +236,6 @@ public static boolean hasParameters(String typeName) { } } - public static String getBaseName(String typeName) { - int idx = typeName.indexOf('('); - if (idx == -1) { - return typeName; - } else { - return typeName.substring(0, idx); - } - } - /** * returns true if both TypeInfos are of primitive type, and the primitive category matches. * @param ti1 @@ -256,7 +243,7 @@ public static String getBaseName(String typeName) { * @return */ public static boolean doPrimitiveCategoriesMatch(TypeInfo ti1, TypeInfo ti2) { - if (ti1.getCategory() == Category.PRIMITIVE && ti2.getCategory() == Category.PRIMITIVE) { + if (ti1.getCategory() == TypeInfo.Category.PRIMITIVE && ti2.getCategory() == TypeInfo.Category.PRIMITIVE) { if (((PrimitiveTypeInfo)ti1).getPrimitiveCategory() == ((PrimitiveTypeInfo)ti2).getPrimitiveCategory()) { return true; @@ -266,328 +253,10 @@ public static boolean doPrimitiveCategoriesMatch(TypeInfo ti1, TypeInfo ti2) { } /** - * Parse a recursive TypeInfo list String. For example, the following inputs - * are valid inputs: - * "int,string,map,list>>,list>" - * The separators between TypeInfos can be ",", ":", or ";". - * - * In order to use this class: TypeInfoParser parser = new - * TypeInfoParser("int,string"); ArrayList typeInfos = - * parser.parseTypeInfos(); - */ - private static class TypeInfoParser { - - private static class Token { - public int position; - public String text; - public boolean isType; - - @Override - public String toString() { - return "" + position + ":" + text; - } - }; - - private static boolean isTypeChar(char c) { - return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == ' ' || c == '$'; - } - - /** - * Tokenize the typeInfoString. The rule is simple: all consecutive - * alphadigits and '_', '.' are in one token, and all other characters are - * one character per token. - * - * tokenize("map") should return - * ["map","<","int",",","string",">"] - * - * Note that we add '$' in new Calcite return path. As '$' will not appear - * in any type in Hive, it is safe to do so. - */ - private static ArrayList tokenize(String typeInfoString) { - ArrayList tokens = new ArrayList(0); - int begin = 0; - int end = 1; - while (end <= typeInfoString.length()) { - // last character ends a token? - // if there are quotes, all the text between the quotes - // is considered a single token (this can happen for - // timestamp with local time-zone) - if (begin > 0 && - typeInfoString.charAt(begin - 1) == '(' && - typeInfoString.charAt(begin) == '\'') { - // Ignore starting quote - begin++; - do { - end++; - } while (typeInfoString.charAt(end) != '\''); - } else if (typeInfoString.charAt(begin) == '\'' && - typeInfoString.charAt(begin + 1) == ')') { - // Ignore closing quote - begin++; - end++; - } - if (end == typeInfoString.length() - || !isTypeChar(typeInfoString.charAt(end - 1)) - || !isTypeChar(typeInfoString.charAt(end))) { - Token t = new Token(); - t.position = begin; - t.text = typeInfoString.substring(begin, end); - t.isType = isTypeChar(typeInfoString.charAt(begin)); - tokens.add(t); - begin = end; - } - end++; - } - return tokens; - } - - public TypeInfoParser(String typeInfoString) { - this.typeInfoString = typeInfoString; - typeInfoTokens = tokenize(typeInfoString); - } - - private final String typeInfoString; - private final ArrayList typeInfoTokens; - private ArrayList typeInfos; - private int iToken; - - public ArrayList parseTypeInfos() { - typeInfos = new ArrayList(); - iToken = 0; - while (iToken < typeInfoTokens.size()) { - typeInfos.add(parseType()); - if (iToken < typeInfoTokens.size()) { - Token separator = typeInfoTokens.get(iToken); - if (",".equals(separator.text) || ";".equals(separator.text) - || ":".equals(separator.text)) { - iToken++; - } else { - throw new IllegalArgumentException( - "Error: ',', ':', or ';' expected at position " - + separator.position + " from '" + typeInfoString + "' " - + typeInfoTokens); - } - } - } - return typeInfos; - } - - private Token peek() { - if (iToken < typeInfoTokens.size()) { - return typeInfoTokens.get(iToken); - } else { - return null; - } - } - - private Token expect(String item) { - return expect(item, null); - } - - private Token expect(String item, String alternative) { - if (iToken >= typeInfoTokens.size()) { - throw new IllegalArgumentException("Error: " + item - + " expected at the end of '" + typeInfoString + "'"); - } - Token t = typeInfoTokens.get(iToken); - if (item.equals("type")) { - if (!serdeConstants.LIST_TYPE_NAME.equals(t.text) - && !serdeConstants.MAP_TYPE_NAME.equals(t.text) - && !serdeConstants.STRUCT_TYPE_NAME.equals(t.text) - && !serdeConstants.UNION_TYPE_NAME.equals(t.text) - && null == PrimitiveObjectInspectorUtils - .getTypeEntryFromTypeName(t.text) - && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else if (item.equals("name")) { - if (!t.isType && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } else { - if (!item.equals(t.text) && !t.text.equals(alternative)) { - throw new IllegalArgumentException("Error: " + item - + " expected at the position " + t.position + " of '" - + typeInfoString + "' but '" + t.text + "' is found."); - } - } - iToken++; - return t; - } - - private String[] parseParams() { - List params = new LinkedList(); - - Token t = peek(); - if (t != null && t.text.equals("(")) { - expect("("); - - // checking for null in the for-loop condition prevents null-ptr exception - // and allows us to fail more gracefully with a parsing error. - for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) { - params.add(expect("name").text); - } - if (params.size() == 0) { - throw new IllegalArgumentException( - "type parameters expected for type string " + typeInfoString); - } - } - - return params.toArray(new String[params.size()]); - } - - private TypeInfo parseType() { - - Token t = expect("type"); - - // Is this a primitive type? - PrimitiveTypeEntry typeEntry = - PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(t.text); - if (typeEntry != null && typeEntry.primitiveCategory != PrimitiveCategory.UNKNOWN ) { - String[] params = parseParams(); - switch (typeEntry.primitiveCategory) { - case CHAR: - case VARCHAR: - if (params == null || params.length == 0) { - throw new IllegalArgumentException(typeEntry.typeName - + " type is specified without length: " + typeInfoString); - } - - int length = 1; - if (params.length == 1) { - length = Integer.parseInt(params[0]); - if (typeEntry.primitiveCategory == PrimitiveCategory.VARCHAR) { - BaseCharUtils.validateVarcharParameter(length); - return TypeInfoFactory.getVarcharTypeInfo(length); - } else { - BaseCharUtils.validateCharParameter(length); - return TypeInfoFactory.getCharTypeInfo(length); - } - } else if (params.length > 1) { - throw new IllegalArgumentException( - "Type " + typeEntry.typeName+ " only takes one parameter, but " + - params.length + " is seen"); - } - - case DECIMAL: - int precision = HiveDecimal.USER_DEFAULT_PRECISION; - int scale = HiveDecimal.USER_DEFAULT_SCALE; - if (params == null || params.length == 0) { - // It's possible that old metadata still refers to "decimal" as a column type w/o - // precision/scale. In this case, the default (10,0) is assumed. Thus, do nothing here. - } else if (params.length == 1) { - // only precision is specified - precision = Integer.valueOf(params[0]); - HiveDecimalUtils.validateParameter(precision, scale); - } else if (params.length == 2) { - // New metadata always have two parameters. - precision = Integer.parseInt(params[0]); - scale = Integer.parseInt(params[1]); - HiveDecimalUtils.validateParameter(precision, scale); - } else if (params.length > 2) { - throw new IllegalArgumentException("Type decimal only takes two parameter, but " + - params.length + " is seen"); - } - return TypeInfoFactory.getDecimalTypeInfo(precision, scale); - - default: - return TypeInfoFactory.getPrimitiveTypeInfo(typeEntry.typeName); - } - } - - // Is this a list type? - if (serdeConstants.LIST_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo listElementType = parseType(); - expect(">"); - return TypeInfoFactory.getListTypeInfo(listElementType); - } - - // Is this a map type? - if (serdeConstants.MAP_TYPE_NAME.equals(t.text)) { - expect("<"); - TypeInfo mapKeyType = parseType(); - expect(","); - TypeInfo mapValueType = parseType(); - expect(">"); - return TypeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); - } - - // Is this a struct type? - if (serdeConstants.STRUCT_TYPE_NAME.equals(t.text)) { - ArrayList fieldNames = new ArrayList(); - ArrayList fieldTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of struct - break; - } - } - Token name = expect("name",">"); - if (name.text.equals(">")) { - break; - } - fieldNames.add(name.text); - expect(":"); - fieldTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); - } - // Is this a union type? - if (serdeConstants.UNION_TYPE_NAME.equals(t.text)) { - List objectTypeInfos = new ArrayList(); - boolean first = true; - do { - if (first) { - expect("<"); - first = false; - } else { - Token separator = expect(">", ","); - if (separator.text.equals(">")) { - // end of union - break; - } - } - objectTypeInfos.add(parseType()); - } while (true); - - return TypeInfoFactory.getUnionTypeInfo(objectTypeInfos); - } - - throw new RuntimeException("Internal error parsing position " - + t.position + " of '" + typeInfoString + "'"); - } - - public PrimitiveParts parsePrimitiveParts() { - PrimitiveParts parts = new PrimitiveParts(); - Token t = expect("type"); - parts.typeName = t.text; - parts.typeParams = parseParams(); - return parts; - } - } - - public static class PrimitiveParts { - public String typeName; - public String[] typeParams; - } - - /** * Make some of the TypeInfo parsing available as a utility. */ public static PrimitiveParts parsePrimitiveParts(String typeInfoString) { - TypeInfoParser parser = new TypeInfoParser(typeInfoString); + TypeInfoParser parser = new TypeInfoParser(typeInfoString, TypeInfoFactory.getInstance()); return parser.parsePrimitiveParts(); } @@ -844,7 +513,7 @@ public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { } public static ArrayList getTypeInfosFromTypeString(String typeString) { - TypeInfoParser parser = new TypeInfoParser(typeString); + TypeInfoParser parser = new TypeInfoParser(typeString, TypeInfoFactory.getInstance()); return parser.parseTypeInfos(); } @@ -861,7 +530,7 @@ public static TypeInfo getTypeInfoFromObjectInspector(ObjectInspector oi) { } public static TypeInfo getTypeInfoFromTypeString(String typeString) { - TypeInfoParser parser = new TypeInfoParser(typeString); + TypeInfoParser parser = new TypeInfoParser(typeString, TypeInfoFactory.getInstance()); return parser.parseTypeInfos().get(0); } @@ -972,7 +641,7 @@ public static boolean implicitConvertible(TypeInfo from, TypeInfo to) { // Reimplemented to use PrimitiveCategory rather than TypeInfo, because // 2 TypeInfos from the same qualified type (varchar, decimal) should still be // seen as equivalent. - if (from.getCategory() == Category.PRIMITIVE && to.getCategory() == Category.PRIMITIVE) { + if (from.getCategory() == TypeInfo.Category.PRIMITIVE && to.getCategory() == TypeInfo.Category.PRIMITIVE) { return implicitConvertible( ((PrimitiveTypeInfo) from).getPrimitiveCategory(), ((PrimitiveTypeInfo) to).getPrimitiveCategory()); diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java deleted file mode 100644 index 842997c021cb16e7695294bf3154028fa7f53068..0000000000000000000000000000000000000000 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.typeinfo; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hive.common.classification.InterfaceAudience; -import org.apache.hadoop.hive.common.classification.InterfaceStability; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; - -/** - * UnionTypeInfo represents the TypeInfo of an union. A union holds only one - * field of the specified fields at any point of time. The fields, a Union can - * hold, can have the same or different TypeInfo. - * - * Always use the TypeInfoFactory to create new TypeInfo objects, instead of - * directly creating an instance of this class. - */ -@InterfaceAudience.Public -@InterfaceStability.Stable -public class UnionTypeInfo extends TypeInfo implements Serializable { - - private static final long serialVersionUID = 1L; - - private List allUnionObjectTypeInfos; - - /** - * For java serialization use only. - */ - public UnionTypeInfo() { - } - - @Override - public String getTypeName() { - StringBuilder sb = new StringBuilder(); - sb.append(serdeConstants.UNION_TYPE_NAME + "<"); - for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) { - if (i > 0) { - sb.append(","); - } - sb.append(allUnionObjectTypeInfos.get(i).getTypeName()); - } - sb.append(">"); - return sb.toString(); - } - - /** - * For java serialization use only. - */ - public void setAllUnionObjectTypeInfos( - List allUnionObjectTypeInfos) { - this.allUnionObjectTypeInfos = allUnionObjectTypeInfos; - } - - /** - * For TypeInfoFactory use only. - */ - UnionTypeInfo(List typeInfos) { - allUnionObjectTypeInfos = new ArrayList(); - allUnionObjectTypeInfos.addAll(typeInfos); - } - - @Override - public Category getCategory() { - return Category.UNION; - } - - public List getAllUnionObjectTypeInfos() { - return allUnionObjectTypeInfos; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - if (!(other instanceof UnionTypeInfo)) { - return false; - } - UnionTypeInfo o = (UnionTypeInfo) other; - - // Compare the field types - return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); - } - - @Override - public int hashCode() { - return allUnionObjectTypeInfos.hashCode(); - } -} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java index edf12a20005896e4e12d0ea6171760a76ee3fe53..a6c248a755becba9469c3ccfd2b3ba841d856144 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java @@ -51,7 +51,6 @@ public boolean equals(Object other) { return this.getLength() == pti.getLength(); } - /** * Generate the hashCode for this TypeInfo. */ diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java b/serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java index 749d8accf7f696aa062ad16cee407aa1d1ef4481..5f7482832e0920fa48d191cf3e91d45ac5e0b957 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.serde2.io.HiveCharWritable; import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; @@ -461,7 +461,7 @@ private void chooseSchema(SupportedTypes supportedTypes, int maxComplexDepth) { } objectInspectorList.add(objectInspector); - if (category == Category.PRIMITIVE) { + if (category == TypeInfo.Category.PRIMITIVE) { } typeNames.add(decoratedTypeName); } @@ -640,7 +640,7 @@ public Object randomWritable(TypeInfo typeInfo, ObjectInspector objectInspector, elementObjectInspector); boolean isStringFamily = false; PrimitiveCategory primitiveCategory = null; - if (elementTypeInfo.getCategory() == Category.PRIMITIVE) { + if (elementTypeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { primitiveCategory = ((PrimitiveTypeInfo) elementTypeInfo).getPrimitiveCategory(); if (primitiveCategory == PrimitiveCategory.STRING || primitiveCategory == PrimitiveCategory.BINARY || diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java index 3736a1f8fcc089469efb79a2c4b22db032b7dc58..650d7d243eee6272347c3326f13d3e40ce90c8c3 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroObjectInspectorGenerator.java @@ -403,7 +403,7 @@ private void verifyMap(final AvroObjectInspectorGenerator aoig, final String fie // Column types assertEquals(1, aoig.getColumnTypes().size()); TypeInfo typeInfo = aoig.getColumnTypes().get(0); - assertEquals(ObjectInspector.Category.MAP, typeInfo.getCategory()); + assertEquals(TypeInfo.Category.MAP, typeInfo.getCategory()); assertTrue(typeInfo instanceof MapTypeInfo); MapTypeInfo mapTypeInfo = (MapTypeInfo)typeInfo; @@ -423,7 +423,7 @@ public void canHandleArrays() throws SerDeException { // Column types assertEquals(1, aoig.getColumnTypes().size()); TypeInfo typeInfo = aoig.getColumnTypes().get(0); - assertEquals(ObjectInspector.Category.LIST, typeInfo.getCategory()); + assertEquals(TypeInfo.Category.LIST, typeInfo.getCategory()); assertTrue(typeInfo instanceof ListTypeInfo); ListTypeInfo listTypeInfo = (ListTypeInfo)typeInfo; @@ -442,7 +442,7 @@ public void canHandleRecords() throws SerDeException { // Column types assertEquals(1, aoig.getColumnTypes().size()); TypeInfo typeInfo = aoig.getColumnTypes().get(0); - assertEquals(ObjectInspector.Category.STRUCT, typeInfo.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, typeInfo.getCategory()); assertTrue(typeInfo instanceof StructTypeInfo); StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo; diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java deleted file mode 100644 index cb7c6ed42158727ab5dd32ecb1f5f7d727b87ba3..0000000000000000000000000000000000000000 --- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.serde2.avro; - -import java.util.Set; -import org.junit.Test; - -import static org.junit.Assert.assertSame; - -public class TestInstanceCache { - private static class Foo { - - private int value = 42; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Foo foo = (Foo) o; - - return value == foo.value; - - } - - @Override - public int hashCode() { - return value; - } - } - - private static class Wrapper { - public final T wrapped; - - private Wrapper(T wrapped) { - this.wrapped = wrapped; - } - } - - @Test - public void instanceCachesOnlyCreateOneInstance() throws AvroSerdeException { - InstanceCache> ic = new InstanceCache>() { - @Override - protected Wrapper makeInstance(Foo hv, - Set seenSchemas) { - return new Wrapper(hv); - } - }; - Foo f1 = new Foo(); - - Wrapper fc = ic.retrieve(f1, null); - assertSame(f1, fc.wrapped); // Our original foo should be in the wrapper - - Foo f2 = new Foo(); // Different instance, same value - - Wrapper fc2 = ic.retrieve(f2, null); - assertSame(fc2,fc); // Since equiv f, should get back first container - assertSame(fc2.wrapped, f1); - } - - @Test - public void instanceCacheReturnsCorrectInstances() throws AvroSerdeException { - InstanceCache> ic = new InstanceCache>() { - @Override - protected Wrapper makeInstance( - String hv, Set seenSchemas) { - return new Wrapper(hv); - } - }; - - Wrapper one = ic.retrieve("one", null); - Wrapper two = ic.retrieve("two", null); - - Wrapper anotherOne = ic.retrieve("one", null); - assertSame(one, anotherOne); - - Wrapper anotherTwo = ic.retrieve("two", null); - assertSame(two, anotherTwo); - } -} diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java index af258c7d174e78458cf8caaa0a58ae903d44c3a8..e5f57107393c7fa82fc011792c323a15752a2950 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java @@ -49,6 +49,7 @@ public void testDisallowRecursiveSchema() + " } ]\n" + "}"; - List types = SchemaToTypeInfo.generateColumnTypes(new Schema.Parser().parse(schemaString)); + List types = SchemaToHiveTypeInfo.getInstance() + .generateColumnTypes(new Schema.Parser().parse(schemaString)); } } \ No newline at end of file diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java index 62741d38d047a1f2cfe1233723ff8ab4c86405d2..da43a3be3ef8d0c6c5d674a3845db13893848417 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; @@ -313,7 +312,7 @@ private void testBinarySortableFast( private void verifyRead(BinarySortableDeserializeRead binarySortableDeserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfo, expectedObject); } else { Object complexFieldObj = VerifyFast.deserializeReadComplexType(binarySortableDeserializeRead, typeInfo); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java index 730764e516513dc26f3708d77d15642baf7e8734..0c417c7bfb385a0b02d4afcf0999ea64144d0ecf 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java @@ -31,11 +31,8 @@ import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.io.ByteWritable; -import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory; -import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap; -import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -518,7 +515,7 @@ public void testLazyUnion() throws Throwable { */ public void testLazyArrayNested() throws Throwable { for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ - testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.LIST); + testNestedinArrayAtLevelExtended(i, TypeInfo.Category.LIST); } } @@ -526,7 +523,7 @@ public void testLazyArrayNested() throws Throwable { * Test the LazyArray class with multiple levels of nesting */ public void testLazyArrayNestedExceedLimit() throws Throwable { - checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.LIST); + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, TypeInfo.Category.LIST); } private void checkExtendedLimitExceeded(int maxLevel, Category type) { @@ -545,7 +542,7 @@ private void checkExtendedLimitExceeded(int maxLevel, Category type) { */ public void testLazyArrayNestedExceedLimitNotExtended() throws Throwable { checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, - ObjectInspector.Category.LIST); + TypeInfo.Category.LIST); } /** @@ -554,7 +551,7 @@ public void testLazyArrayNestedExceedLimitNotExtended() throws Throwable { */ public void testLazyMapNestedExceedLimitNotExtended() throws Throwable { checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD-1, - ObjectInspector.Category.MAP); + TypeInfo.Category.MAP); } /** @@ -563,7 +560,7 @@ public void testLazyMapNestedExceedLimitNotExtended() throws Throwable { */ public void testLazyStructNestedExceedLimitNotExtended() throws Throwable { checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, - ObjectInspector.Category.STRUCT); + TypeInfo.Category.STRUCT); } /** @@ -572,7 +569,7 @@ public void testLazyStructNestedExceedLimitNotExtended() throws Throwable { */ public void testLazyUnionNestedExceedLimitNotExtended() throws Throwable { checkNotExtendedLimitExceeded(DEFAULT_LEVEL_THRESHOLD, - ObjectInspector.Category.UNION); + TypeInfo.Category.UNION); } private void checkNotExtendedLimitExceeded(int maxLevel, Category type) { @@ -591,7 +588,7 @@ private void checkNotExtendedLimitExceeded(int maxLevel, Category type) { public void testLazyMapNested() throws Throwable { //map max nesting level is one less because it uses an additional separator for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD - 1; i++ ){ - testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.MAP); + testNestedinArrayAtLevelExtended(i, TypeInfo.Category.MAP); } } @@ -600,7 +597,7 @@ public void testLazyMapNested() throws Throwable { */ public void testLazyMapNestedExceedLimit() throws Throwable { //map max nesting level is one less because it uses an additional separator - checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD - 1, ObjectInspector.Category.MAP); + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD - 1, TypeInfo.Category.MAP); } /** @@ -608,7 +605,7 @@ public void testLazyMapNestedExceedLimit() throws Throwable { */ public void testLazyUnionNested() throws Throwable { for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ - testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.UNION); + testNestedinArrayAtLevelExtended(i, TypeInfo.Category.UNION); } } @@ -616,7 +613,7 @@ public void testLazyUnionNested() throws Throwable { * Test the LazyUnion class with multiple levels of nesting */ public void testLazyUnionNestedExceedLimit() throws Throwable { - checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.UNION); + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, TypeInfo.Category.UNION); } /** @@ -624,7 +621,7 @@ public void testLazyUnionNestedExceedLimit() throws Throwable { */ public void testLazyStructNested() throws Throwable { for(int i = 2; i < EXTENDED_LEVEL_THRESHOLD; i++ ){ - testNestedinArrayAtLevelExtended(i, ObjectInspector.Category.STRUCT); + testNestedinArrayAtLevelExtended(i, TypeInfo.Category.STRUCT); } } @@ -636,7 +633,7 @@ public void testLazyStructNested() throws Throwable { * @throws SerDeException */ private void testNestedinArrayAtLevelExtended(int nestingLevel, - ObjectInspector.Category dtype) throws SerDeException { + Category dtype) throws SerDeException { Properties tableProp = new Properties(); tableProp.setProperty(LazySerDeParameters.SERIALIZATION_EXTEND_NESTING_LEVELS, "true"); testNestedinArrayAtLevel(nestingLevel, dtype, tableProp); @@ -646,7 +643,7 @@ private void testNestedinArrayAtLevelExtended(int nestingLevel, * Test the LazyStruct class with multiple levels of nesting */ public void testLazyStructNestedExceedLimit() throws Throwable { - checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, ObjectInspector.Category.STRUCT); + checkExtendedLimitExceeded(EXTENDED_LEVEL_THRESHOLD, TypeInfo.Category.STRUCT); } /** @@ -656,7 +653,7 @@ public void testLazyStructNestedExceedLimit() throws Throwable { * @throws SerDeException */ private void testNestedinArrayAtLevel(int nestingLevel, - ObjectInspector.Category dtype, Properties tableProp) throws SerDeException { + Category dtype, Properties tableProp) throws SerDeException { //create type with nestingLevel levels of nesting //set inner schema for dtype diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java index fbb6040331c656d743b4599f9e303b08d0012c4c..92d422687135b840744aa4e1531ac54552ea9665 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.serde2.lazy; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Properties; import java.util.Random; @@ -35,7 +34,6 @@ import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; @@ -199,7 +197,7 @@ private void testLazySimpleFast( private void verifyReadNull(LazySimpleDeserializeRead lazySimpleDeserializeRead, TypeInfo typeInfo) throws IOException { - if (typeInfo.getCategory() == Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, null); } else { Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo); @@ -211,7 +209,7 @@ private void verifyReadNull(LazySimpleDeserializeRead lazySimpleDeserializeRead, private void verifyRead(LazySimpleDeserializeRead lazySimpleDeserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, expectedObject); } else { Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java index e95c6eb4350f4dd7e574a62d5642c2096437f666..6b2281165381ac26415c7821f33a31fb7ec8aa1d 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.serde2.lazybinary; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Random; @@ -32,13 +31,11 @@ import org.apache.hadoop.hive.serde2.lazy.VerifyLazy; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead; import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.io.BytesWritable; -import org.apache.hadoop.io.Writable; public class TestLazyBinaryFast extends TestCase { @@ -211,7 +208,7 @@ private void testLazyBinaryFast( private void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead, TypeInfo typeInfo, Object expectedObject) throws IOException { - if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) { + if (typeInfo.getCategory() == TypeInfo.Category.PRIMITIVE) { VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject); } else { Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazyBinaryDeserializeRead, typeInfo); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java index eca74057a30b11e3462bbe3cc468a6159a512ca4..ad3139f21232481bb68ddd36a617b27841ffb830 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java @@ -23,10 +23,10 @@ import junit.framework.TestCase; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.serde2.thrift.test.IntString; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * TestObjectInspectorUtils. @@ -63,7 +63,7 @@ public void testObjectInspectorUtils() throws Throwable { ObjectInspectorFactory.ObjectInspectorOptions.THRIFT); // metadata - assertEquals(Category.STRUCT, oi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, oi1.getCategory()); // standard ObjectInspector StructObjectInspector soi = (StructObjectInspector) ObjectInspectorUtils .getStandardObjectInspector(oi1); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java index b4efdf865f54f61439e445f37b3b532cf8dcc1e8..26c5cbf666722a439e36db3ee302b6af6e06d376 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestProtocolBuffersObjectInspectors.java @@ -23,10 +23,10 @@ import junit.framework.TestCase; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.proto.test.Complexpb.Complex; import org.apache.hadoop.hive.serde2.proto.test.Complexpb.IntString; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * TestProtocolBuffersObjectInspectors. @@ -46,7 +46,7 @@ public void testProtocolBuffersObjectInspectors() throws Throwable { assertEquals(oi1, oi2); // metadata - assertEquals(Category.STRUCT, oi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, oi1.getCategory()); StructObjectInspector soi = (StructObjectInspector) oi1; List fields = soi.getAllStructFieldRefs(); assertEquals(5, fields.size()); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java index cb1440e98b8fad399c4c02a7440d2a30483455f4..c9f4f2d697b9373e36143c0e6a8ce32af6d88ac9 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java @@ -30,11 +30,11 @@ import org.apache.commons.lang.mutable.MutableObject; import org.apache.hadoop.hive.common.ObjectPair; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import junit.framework.TestCase; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * TestReflectionObjectInspectors. @@ -53,7 +53,7 @@ public void testReflectionObjectInspectors() throws Throwable { assertEquals(oi1, oi2); // metadata - assertEquals(Category.STRUCT, oi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, oi1.getCategory()); StructObjectInspector soi = (StructObjectInspector) oi1; List fields = soi.getAllStructFieldRefs(); assertEquals(6, fields.size()); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java index 17b844c948532faf9c25bfd1a3174eb251026d89..20acb9d9996f69a822c813f0276a5cb02ced4c08 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestStandardObjectInspectors.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable; import org.apache.hadoop.hive.serde2.io.ShortWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -64,7 +63,7 @@ void doTestStandardPrimitiveObjectInspector(Class writableClass, .getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils .getTypeEntryFromPrimitiveWritableClass(writableClass).primitiveCategory); assertEquals(oi1, oi2); - assertEquals(Category.PRIMITIVE, oi1.getCategory()); + assertEquals(TypeInfo.Category.PRIMITIVE, oi1.getCategory()); assertEquals(writableClass, oi1.getPrimitiveWritableClass()); assertEquals(javaClass, oi1.getJavaPrimitiveClass()); // Cannot create NullWritable instances @@ -112,7 +111,7 @@ void doTestJavaPrimitiveObjectInspector(Class writableClass, .getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils .getTypeEntryFromPrimitiveJavaClass(javaClass).primitiveCategory); assertEquals(oi1, oi2); - assertEquals(Category.PRIMITIVE, oi1.getCategory()); + assertEquals(TypeInfo.Category.PRIMITIVE, oi1.getCategory()); assertEquals(javaClass, oi1.getJavaPrimitiveClass()); assertEquals(writableClass, oi1.getPrimitiveWritableClass()); if (javaObject != null) { @@ -172,7 +171,7 @@ public void testStandardListObjectInspector() throws Throwable { assertEquals(loi1, loi2); // metadata - assertEquals(Category.LIST, loi1.getCategory()); + assertEquals(TypeInfo.Category.LIST, loi1.getCategory()); assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, loi1 .getListElementObjectInspector()); @@ -231,7 +230,7 @@ public void testStandardMapObjectInspector() throws Throwable { assertEquals(moi1, moi2); // metadata - assertEquals(Category.MAP, moi1.getCategory()); + assertEquals(TypeInfo.Category.MAP, moi1.getCategory()); assertEquals(moi1.getMapKeyObjectInspector(), PrimitiveObjectInspectorFactory.javaStringObjectInspector); assertEquals(moi2.getMapValueObjectInspector(), @@ -332,7 +331,7 @@ private void doStandardObjectInspectorTest(boolean testComments) { assertEquals(soi1, soi2); // metadata - assertEquals(Category.STRUCT, soi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, soi1.getCategory()); List fields = soi1.getAllStructFieldRefs(); assertEquals(3, fields.size()); for (int i = 0; i < 3; i++) { @@ -440,7 +439,7 @@ public void testStandardUnionObjectInspector() throws Throwable { .getStandardUnionObjectInspector(ois))); // metadata - assertEquals(Category.UNION, uoi1.getCategory()); + assertEquals(TypeInfo.Category.UNION, uoi1.getCategory()); List uois = uoi1.getObjectInspectors(); assertEquals(6, uois.size()); for (int i = 0; i < 6; i++) { @@ -458,7 +457,7 @@ public void testStandardUnionObjectInspector() throws Throwable { assertEquals(unionTypeName.toString(), uoi1.getTypeName()); // TypeInfo TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1); - assertEquals(Category.UNION, typeInfo1.getCategory()); + assertEquals(TypeInfo.Category.UNION, typeInfo1.getCategory()); assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName()); assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName()); assertEquals(typeInfo1, diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java index de291fa6d8a82e8726481e940bf3f7c4f55bd447..9101a53898c1e5e233b717b907dbca89174b3d4f 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Set; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import org.apache.hadoop.hive.serde2.thrift.test.IntString; @@ -31,6 +30,7 @@ import org.apache.hadoop.hive.serde2.thrift.test.SetIntString; import junit.framework.TestCase; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * TestThriftObjectInspectors. @@ -50,7 +50,7 @@ public void testThriftObjectInspectors() throws Throwable { assertEquals(oi1, oi2); // metadata - assertEquals(Category.STRUCT, oi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, oi1.getCategory()); StructObjectInspector soi = (StructObjectInspector) oi1; List fields = soi.getAllStructFieldRefs(); assertEquals(10, fields.size()); @@ -141,7 +141,7 @@ public void testThriftSetObjectInspector() throws Throwable { assertEquals(oi1, oi2); // metadata - assertEquals(Category.STRUCT, oi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, oi1.getCategory()); StructObjectInspector soi = (StructObjectInspector) oi1; List fields = soi.getAllStructFieldRefs(); assertEquals(2, fields.size()); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java index efc96931caee9314be29c74b1de746740718ee95..6249d1d8aa1e2323b7bc0b5b81b37a93a6a662f6 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestUnionStructObjectInspector.java @@ -23,8 +23,8 @@ import junit.framework.TestCase; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; /** * TestUnionStructObjectInspector. @@ -69,7 +69,7 @@ public void testUnionStructObjectInspector() throws Throwable { assertEquals(usoi1, usoi2); // metadata - assertEquals(Category.STRUCT, usoi1.getCategory()); + assertEquals(TypeInfo.Category.STRUCT, usoi1.getCategory()); List fields = usoi1.getAllStructFieldRefs(); assertEquals(5, fields.size()); for (int i = 0; i < 5; i++) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AvroStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AvroStorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..ed3038d91d2537122bddb797a0a95f54e3ce4d17 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/AvroStorageSchemaReader.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public class AvroStorageSchemaReader implements StorageSchemaReader { + private static final Logger LOG = LoggerFactory.getLogger(AvroStorageSchemaReader.class); + + @Override + public List readSchema(Table tbl, EnvironmentContext envContext, + Configuration conf) throws MetaException { + Properties tblMetadataProperties = MetaStoreUtils.getTableMetadata(tbl); + try { + return AvroSchemaUtils.getFieldsFromAvroSchema(conf, tblMetadataProperties); + } catch (AvroSerdeException e) { + LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } catch (IOException e) { + LOG.warn("Received IOException while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java index d5dea4dc3ca6a83a863326e1c75e2480898d00af..ab8590e1edc1ed14baf6bce086450bc1971f9831 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -72,6 +72,8 @@ public static final String TIMESTAMPTZ_TYPE_NAME = "timestamp with time zone"; + public static final String TIMESTAMPLOCALTZ_TYPE_NAME = "timestamp with local time zone"; + public static final String LIST_TYPE_NAME = "array"; public static final String MAP_TYPE_NAME = "map"; @@ -105,7 +107,8 @@ INTERVAL_DAY_TIME_TYPE_NAME, DECIMAL_TYPE_NAME, BINARY_TYPE_NAME, - TIMESTAMPTZ_TYPE_NAME); + TIMESTAMPTZ_TYPE_NAME, + TIMESTAMPLOCALTZ_TYPE_NAME); public static final Set StringTypes = StringUtils.asSet( STRING_TYPE_NAME, diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java index 1dbfa4272cd5368242d335fbde564d35cac26bba..081163a92db2a5e111348501eaf47e90c753cb89 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/DefaultStorageSchemaReader.java @@ -18,21 +18,130 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.ColumnType; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.utils.StorageSchemaUtils; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeException; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Properties; + +import static org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils.LIST_COLUMN_COMMENTS; /** * Default StorageSchemaReader. This just throws as the metastore currently doesn't know how to * read schemas from storage. */ public class DefaultStorageSchemaReader implements StorageSchemaReader { + private final static Logger LOG = LoggerFactory.getLogger(DefaultStorageSchemaReader.class); + + private static final String AVRO_SERIALIZATION_LIB = + "org.apache.hadoop.hive.serde2.avro.AvroSerDe"; + @Override public List readSchema(Table tbl, EnvironmentContext envContext, - Configuration conf) throws MetaException { - throw new UnsupportedOperationException("Storage schema reading not supported"); + Configuration conf) throws MetaException { + String serializationLib = tbl.getSd().getSerdeInfo().getSerializationLib(); + if (null == serializationLib || MetastoreConf + .getStringCollection(conf, MetastoreConf.ConfVars.SERDES_USING_METASTORE_FOR_SCHEMA) + .contains(serializationLib)) { + //safety check to make sure we should be using storage schema reader for this table + throw new MetaException( + "Invalid usage of default storage schema reader for table " + tbl.getTableName() + + " with storage descriptor " + tbl.getSd().getSerdeInfo().getSerializationLib()); + } + Properties tblMetadataProperties = MetaStoreUtils.getTableMetadata(tbl); + if(AVRO_SERIALIZATION_LIB.equals(serializationLib)) { + //in case of avro table use AvroStorageSchemaReader utils + try { + return AvroSchemaUtils.getFieldsFromAvroSchema(conf, tblMetadataProperties); + } catch (AvroSerdeException e) { + LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } catch (IOException e) { + LOG.warn("Exception received while reading avro schema for table " + tbl.getTableName(), e); + throw new MetaException(e.getMessage()); + } + } else { + return getFieldSchemasFromTableMetadata(tblMetadataProperties); + } + } + + /** + * This method implements a generic way to get the FieldSchemas from the table metadata + * properties like column names and column types. Most of the serdes have the same implemention + * in their initialize method + * //TODO refactor the common code from the serdes and move it to serde-api so that there is no + * //duplicate code + * + * @return list of FieldSchema objects + */ + public static List getFieldSchemasFromTableMetadata( + Properties tblMetadataProperties) { + List columnNames = null; + List columnTypes = null; + // Get column names and types + String columnNameProperty = tblMetadataProperties.getProperty( ColumnType.LIST_COLUMNS); + String columnTypeProperty = tblMetadataProperties.getProperty( ColumnType.LIST_COLUMN_TYPES); + final String columnNameDelimiter = tblMetadataProperties + .containsKey( ColumnType.COLUMN_NAME_DELIMITER) ? tblMetadataProperties + .getProperty( ColumnType.COLUMN_NAME_DELIMITER) : String + .valueOf(StorageSchemaUtils.COMMA); + // all table column names + if (columnNameProperty.isEmpty()) { + columnNames = Collections.emptyList(); + } else { + columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter)); + } + + // all column types + if (columnTypeProperty.isEmpty()) { + columnTypes = Collections.emptyList(); + } else { + columnTypes = StorageSchemaUtils.getTypeInfosFromTypeString(columnTypeProperty); + } + + final String columnCommentProperty = + tblMetadataProperties.getProperty(LIST_COLUMN_COMMENTS, ""); + List columnComments = null; + if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { + columnComments = new ArrayList<>(0); + } else { + columnComments = Arrays.asList( + columnCommentProperty.split(String.valueOf(ColumnType.COLUMN_COMMENTS_DELIMITER))); + } + LOG.debug("columns: {}, {}", columnNameProperty, columnNames); + LOG.debug("types: {}, {} ", columnTypeProperty, columnTypes); + LOG.debug("comments: {} ", columnCommentProperty); + return getFieldSchemaFromColumnInfo(columnNames, columnTypes, columnComments); + } + + private static List getFieldSchemaFromColumnInfo(List columnNames, + List columnTypes, List columnComments) { + int len = columnNames.size(); + List fieldSchemas = new ArrayList<>(len); + for (int i = 0; i < len; i++) { + FieldSchema fieldSchema = new FieldSchema(); + fieldSchema.setName(columnNames.get(i)); + //In case of complex types getTypeName() will recusively go into typeName + //of individual fields when the ColumnType was constructed + //in SchemaToTypeInfo.generateColumnTypes in the constructor + fieldSchema.setType(columnTypes.get(i).getTypeName()); + fieldSchema.setComment(StorageSchemaUtils.determineFieldComment(columnComments.get(i))); + } + return fieldSchemas; } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java index 6251e23991b8f898f939f5848fc5cbf5e8ceb07c..aff9bc89e2d43a92561b6e24c4d9d158644964af 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/StorageSchemaReader.java @@ -31,8 +31,7 @@ * An interface to implement reading schemas from stored data. */ @InterfaceAudience.Public -@InterfaceStability.Evolving -interface StorageSchemaReader { +@InterfaceStability.Evolving public interface StorageSchemaReader { /** * Read the schema from the storage representation of the table. * @param tbl metastore table object diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/AvroFieldSchemaGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/AvroFieldSchemaGenerator.java new file mode 100644 index 0000000000000000000000000000000000000000..d270fe5fe47000828e2cb7a4c92fca6d7ac3a2c3 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/AvroFieldSchemaGenerator.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.avro.Schema; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeException; +import org.apache.hadoop.hive.serde2.avro.SchemaToMetastoreTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class AvroFieldSchemaGenerator { + final private List columnNames; + final private List columnTypes; + final private List columnComments; + + public AvroFieldSchemaGenerator(Schema schema) throws AvroSerdeException { + verifySchemaIsARecord(schema); + + this.columnNames = generateColumnNames(schema); + this.columnTypes = SchemaToMetastoreTypeInfo.getInstance().generateColumnTypes(schema); + this.columnComments = generateColumnComments(schema); + assert columnNames.size() == columnTypes.size(); + } + + private static void verifySchemaIsARecord(Schema schema) throws AvroSerdeException { + if(!schema.getType().equals(Schema.Type.RECORD)) { + throw new AvroSerdeException("Schema for table must be of type RECORD. " + + "Received type: " + schema.getType()); + } + } + + private static List generateColumnNames(Schema schema) { + List fields = schema.getFields(); + List fieldsList = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + fieldsList.add(field.name()); + } + + return fieldsList; + } + + private static List generateColumnComments(Schema schema) { + List fields = schema.getFields(); + List fieldComments = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + String fieldComment = field.doc() == null ? "" : field.doc(); + fieldComments.add(fieldComment); + } + + return fieldComments; + } + + public List getFieldSchemas() throws AvroSerdeException { + int len = columnNames.size(); + List fieldSchemas = new ArrayList<>(len); + for(int i = 0; i getFieldsFromAvroSchema(Configuration configuration, + Properties properties) throws AvroSerdeException, IOException { + // Reset member variables so we don't get in a half-constructed state + Schema schema = null; + List columnNames = null; + List columnTypes = null; + + final String columnNameProperty = properties.getProperty(ColumnType.LIST_COLUMNS); + final String columnTypeProperty = properties.getProperty(ColumnType.LIST_COLUMN_TYPES); + final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS,""); + final String columnNameDelimiter = properties.containsKey(ColumnType.COLUMN_NAME_DELIMITER) ? properties + .getProperty(ColumnType.COLUMN_NAME_DELIMITER) : String.valueOf(COMMA); + + if (hasExternalSchema(properties) + || columnNameProperty == null || columnNameProperty.isEmpty() + || columnTypeProperty == null || columnTypeProperty.isEmpty()) { + schema = AvroSchemaUtils.determineSchemaOrThrowException(configuration, properties); + } else { + // Get column names and sort order + columnNames = StringUtils.intern( + Arrays.asList(columnNameProperty.split(columnNameDelimiter))); + columnTypes = new TypeInfoParser(columnTypeProperty, MetastoreTypeInfoFactory + .getInstance()).parseTypeInfos(); + + schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty); + properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Avro schema is " + schema); + } + + if (configuration == null) { + LOG.debug("Configuration null, not inserting schema"); + } else { + configuration.set( + AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false)); + } + return new AvroFieldSchemaGenerator(schema).getFieldSchemas(); + } + + + private static boolean hasExternalSchema(Properties properties) { + return properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()) != null + || properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName()) != null; + } + + public static boolean supportedCategories(TypeInfo ti) { + final Category c = ti.getCategory(); + return c.equals(TypeInfo.Category.PRIMITIVE) || + c.equals(TypeInfo.Category.MAP) || + c.equals(TypeInfo.Category.LIST) || + c.equals(TypeInfo.Category.STRUCT) || + c.equals(TypeInfo.Category.UNION); + } + + /** + * Attempt to determine the schema via the usual means, but do not throw + * an exception if we fail. Instead, signal failure via a special + * schema. + */ + public static Schema determineSchemaOrReturnErrorSchema(Configuration conf, Properties props) { + try { + return AvroSchemaUtils.determineSchemaOrThrowException(conf, props); + } catch(AvroSerdeException he) { + LOG.warn("Encountered AvroSerdeException determining schema. Returning " + + "signal schema to indicate problem", he); + } catch (Exception e) { + LOG.warn("Encountered exception determining schema. Returning signal " + + "schema to indicate problem", e); + } + return SchemaResolutionProblem.SIGNAL_BAD_SCHEMA; + } + + /** + * Determine the schema to that's been provided for Avro serde work. + * @param properties containing a key pointing to the schema, one way or another + * @return schema to use while serdeing the avro file + * @throws IOException if error while trying to read the schema from another location + * @throws AvroSerdeException if unable to find a schema or pointer to it in the properties + */ + public static Schema determineSchemaOrThrowException(Configuration conf, Properties properties) + throws IOException, AvroSerdeException { + String schemaString = properties.getProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName()); + if(schemaString != null && !schemaString.equals(SCHEMA_NONE)) + return AvroSchemaUtils.getSchemaFor(schemaString); + + // Try pulling directly from URL + schemaString = properties.getProperty(AvroTableProperties.SCHEMA_URL.getPropName()); + if (schemaString == null) { + final String columnNameProperty = properties.getProperty(ColumnType.LIST_COLUMNS); + final String columnTypeProperty = properties.getProperty(ColumnType.LIST_COLUMN_TYPES); + final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS); + if (columnNameProperty == null || columnNameProperty.isEmpty() + || columnTypeProperty == null || columnTypeProperty.isEmpty() ) { + throw new AvroSerdeException(EXCEPTION_MESSAGE); + } + final String columnNameDelimiter = properties.containsKey(ColumnType.COLUMN_NAME_DELIMITER) ? properties + .getProperty(ColumnType.COLUMN_NAME_DELIMITER) : String.valueOf(COMMA); + // Get column names and types + List columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter)); + List columnTypes = + new TypeInfoParser(columnTypeProperty, + MetastoreTypeInfoFactory.getInstance()).parseTypeInfos(); + //TODO Why can't we directly bypass this whole logic and use ColumnTypeInfo to use + //AvroFieldSchemaGenerator directly? + Schema schema = getSchemaFromCols(properties, columnNames, columnTypes, columnCommentProperty); + properties.setProperty(AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema.toString()); + if (conf != null) + conf.set(AvroTableProperties.AVRO_SERDE_SCHEMA.getPropName(), schema.toString(false)); + return schema; + } else if(schemaString.equals(SCHEMA_NONE)) { + throw new AvroSerdeException(EXCEPTION_MESSAGE); + } + + try { + Schema s = getSchemaFromFS(schemaString, conf); + if (s == null) { + //in case schema is not a file system + return AvroSchemaUtils.getSchemaFor(new URL(schemaString)); + } + return s; + } catch (IOException ioe) { + throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, ioe); + } catch (URISyntaxException urie) { + throw new AvroSerdeException("Unable to read schema from given path: " + schemaString, urie); + } + } + + // Protected for testing and so we can pass in a conf for testing. + protected static Schema getSchemaFromFS(String schemaFSUrl, + Configuration conf) throws IOException, URISyntaxException { + FSDataInputStream in = null; + FileSystem fs = null; + try { + fs = FileSystem.get(new URI(schemaFSUrl), conf); + } catch (IOException ioe) { + //return null only if the file system in schema is not recognized + if (LOG.isDebugEnabled()) { + String msg = "Failed to open file system for uri " + schemaFSUrl + " assuming it is not a FileSystem url"; + LOG.debug(msg, ioe); + } + + return null; + } + try { + in = fs.open(new Path(schemaFSUrl)); + Schema s = AvroSchemaUtils.getSchemaFor(in); + return s; + } finally { + if(in != null) in.close(); + } + } + + public static Schema getSchemaFor(File file) { + Schema.Parser parser = new Schema.Parser(); + Schema schema; + try { + schema = parser.parse(file); + } catch (IOException e) { + throw new RuntimeException("Failed to parse Avro schema from " + file.getName(), e); + } + return schema; + } + + public static Schema getSchemaFor(InputStream stream) { + Schema.Parser parser = new Schema.Parser(); + Schema schema; + try { + schema = parser.parse(stream); + } catch (IOException e) { + throw new RuntimeException("Failed to parse Avro schema", e); + } + return schema; + } + + public static Schema getSchemaFor(String str) { + Schema.Parser parser = new Schema.Parser(); + Schema schema = parser.parse(str); + return schema; + } + + public static Schema getSchemaFor(URL url) { + InputStream in = null; + try { + in = url.openStream(); + return getSchemaFor(in); + } catch (Exception e) { + throw new RuntimeException("Failed to parse Avro schema", e); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Ignore + } + } + } + } + + public static Schema getSchemaFromCols(Properties properties, + List columnNames, List columnTypes, String columnCommentProperty) { + List columnComments; + if (columnCommentProperty == null || columnCommentProperty.isEmpty()) { + columnComments = new ArrayList(); + } else { + //Comments are separated by "\0" in columnCommentProperty, see method getSchema + //in MetaStoreUtils where this string columns.comments is generated + columnComments = Arrays.asList(columnCommentProperty.split("\0")); + + if (LOG.isDebugEnabled()) { + LOG.debug("columnComments is " + columnCommentProperty); + } + } + if (columnNames.size() != columnTypes.size()) { + throw new IllegalArgumentException("getSchemaFromCols initialization failed. Number of column " + + "name and column type differs. columnNames = " + columnNames + ", columnTypes = " + + columnTypes); + } + + final String tableName = properties.getProperty(AvroSerDeConstants.TABLE_NAME); + final String tableComment = properties.getProperty(AvroSerDeConstants.TABLE_COMMENT); + TypeInfoToSchema metastoreTypeInfoToSchema = new TypeInfoToSchema(); + return metastoreTypeInfoToSchema.convert(columnNames, columnTypes, columnComments, + properties.getProperty(AvroTableProperties.SCHEMA_NAMESPACE.getPropName()), + properties.getProperty(AvroTableProperties.SCHEMA_NAME.getPropName(), tableName), + properties.getProperty(AvroTableProperties.SCHEMA_DOC.getPropName(), tableComment)); + + } + + /** + * Determine if an Avro schema is of type Union[T, NULL]. Avro supports nullable + * types via a union of type T and null. This is a very common use case. + * As such, we want to silently convert it to just T and allow the value to be null. + * + * When a Hive union type is used with AVRO, the schema type becomes + * Union[NULL, T1, T2, ...]. The NULL in the union should be silently removed + * + * @return true if type represents Union[T, Null], false otherwise + */ + public static boolean isNullableType(Schema schema) { + if (!schema.getType().equals(Schema.Type.UNION)) { + return false; + } + + List itemSchemas = schema.getTypes(); + if (itemSchemas.size() < 2) { + return false; + } + + for (Schema itemSchema : itemSchemas) { + if (Schema.Type.NULL.equals(itemSchema.getType())) { + return true; + } + } + + // [null, null] not allowed, so this check is ok. + return false; + } + + /** + * In a nullable type, get the schema for the non-nullable type. This method + * does no checking that the provides Schema is nullable. + */ + public static Schema getOtherTypeFromNullableType(Schema schema) { + List itemSchemas = new ArrayList<>(); + for (Schema itemSchema : schema.getTypes()) { + if (!Schema.Type.NULL.equals(itemSchema.getType())) { + itemSchemas.add(itemSchema); + } + } + + if (itemSchemas.size() > 1) { + return Schema.createUnion(itemSchemas); + } else { + return itemSchemas.get(0); + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StorageSchemaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StorageSchemaUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..5ec642f9b43e121de24c19970e06a94e735ad744 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StorageSchemaUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoParser; + +import java.util.List; + +public class StorageSchemaUtils { + public static final char COMMA = ','; + public static List getTypeInfosFromTypeString(String columnTypeProperty) { + return new TypeInfoParser(columnTypeProperty, MetastoreTypeInfoFactory.getInstance()) + .parseTypeInfos(); + } + + private static final String FROM_STORAGE_SCHEMA_READER = "generated by storage schema reader"; + public static String determineFieldComment(String comment) { + return (comment == null) ? FROM_STORAGE_SCHEMA_READER : comment; + } +} \ No newline at end of file diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..42868ea0f6cb6c6259e9b170d0788582337a68ab --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerDeConstants.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +/** + * This class contains some of the constants which are specific to AvroSerDe + * They should always match with the constants defined in AvroSerDe.java in Hive Source code. These + * constants were copied as part of separating metastore from Hive. + */ +public class AvroSerDeConstants { + public static final String TABLE_NAME = "name"; + public static final String TABLE_COMMENT = "comment"; + public static final String LIST_COLUMN_COMMENTS = "columns.comments"; + + //it just so happens that the AVRO has these constants which are same as defined in ColumnType + //We should still keep it separate in case in future we need to separate the two + public static final String DECIMAL_TYPE_NAME = "decimal"; + public static final String CHAR_TYPE_NAME = "char"; + public static final String VARCHAR_TYPE_NAME = "varchar"; + public static final String DATE_TYPE_NAME = "date"; + + public static final String AVRO_TIMESTAMP_TYPE_NAME = "timestamp-millis"; + public static final String AVRO_PROP_LOGICAL_TYPE = "logicalType"; + public static final String AVRO_PROP_PRECISION = "precision"; + public static final String AVRO_PROP_SCALE = "scale"; + public static final String AVRO_PROP_MAX_LENGTH = "maxLength"; + public static final String AVRO_STRING_TYPE_NAME = "string"; + public static final String AVRO_INT_TYPE_NAME = "int"; + public static final String AVRO_LONG_TYPE_NAME = "long"; +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java new file mode 100644 index 0000000000000000000000000000000000000000..bbf4cc1078f2a9d1395723c395228db865ca850d --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/AvroSerdeException.java @@ -0,0 +1,11 @@ +package org.apache.hadoop.hive.serde2.avro; + +public class AvroSerdeException extends Exception { + public AvroSerdeException(String s, Exception ex) { + super(s, ex); + } + + public AvroSerdeException(String msg) { + super(msg); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java new file mode 100644 index 0000000000000000000000000000000000000000..2d5202091505990f46586b67f0243193db2a2253 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/InstanceCache.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Cache for objects whose creation only depends on some other set of objects and therefore can be + * used against other equivalent versions of those objects. Essentially memoizes instance creation. + * + * @param Object that determines the instance. The cache uses this object as a key for + * its hash which is why it is imperative to have appropriate equals and hashcode + * implementation for this object for the cache to work properly + * @param Instance that will be created from SeedObject. + */ +public abstract class InstanceCache { + private static final Logger LOG = LoggerFactory.getLogger(InstanceCache.class); + Map cache = new HashMap(); + + public InstanceCache() {} + + /** + * Retrieve (or create if it doesn't exist) the correct Instance for this + * SeedObject + */ + public Instance retrieve(SeedObject hv) throws AvroSerdeException { + return retrieve(hv, null); + } + + /** + * Retrieve (or create if it doesn't exist) the correct Instance for this + * SeedObject using 'seenSchemas' to resolve circular references + */ + public synchronized Instance retrieve(SeedObject hv, + Set seenSchemas) throws AvroSerdeException { + if(LOG.isDebugEnabled()) LOG.debug("Checking for hv: " + hv.toString()); + + if(cache.containsKey(hv)) { + if(LOG.isDebugEnabled()) LOG.debug("Returning cache result."); + return cache.get(hv); + } + + if(LOG.isDebugEnabled()) LOG.debug("Creating new instance and storing in cache"); + + Instance instance = makeInstance(hv, seenSchemas); + cache.put(hv, instance); + return instance; + } + + protected abstract Instance makeInstance(SeedObject hv, + Set seenSchemas) throws AvroSerdeException; +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java new file mode 100644 index 0000000000000000000000000000000000000000..3470514c3a059a2dd4faa51173f781d778ab02b0 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaResolutionProblem.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.avro.Schema; +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; + +@LimitedPrivate("Hive") +public class SchemaResolutionProblem { + static final String sentinelString = "{\n" + + " \"namespace\": \"org.apache.hadoop.hive\",\n" + + " \"name\": \"CannotDetermineSchemaSentinel\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"ERROR_ERROR_ERROR_ERROR_ERROR_ERROR_ERROR\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"Cannot_determine_schema\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"check\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"schema\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"url\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"and\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"literal\",\n" + + " \"type\":\"string\"\n" + + " }\n" + + " ]\n" + + "}"; + public final static Schema SIGNAL_BAD_SCHEMA = AvroSchemaUtils.getSchemaFor(sentinelString); +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToMetastoreTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToMetastoreTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..b40a2411a86b7c76bb01c9b0f81007ec3af55a89 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToMetastoreTypeInfo.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeInfoFactory; + +public class SchemaToMetastoreTypeInfo extends SchemaToTypeInfo { + private static final SchemaToMetastoreTypeInfo instance = new SchemaToMetastoreTypeInfo(); + private SchemaToMetastoreTypeInfo() { + super(MetastoreTypeInfoFactory.getInstance()); + } + + public static final SchemaToMetastoreTypeInfo getInstance() { + return instance; + } +} \ No newline at end of file diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..61433a596fe65bc484701a857d1c240b99ef42c0 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import static org.apache.avro.Schema.Type.BOOLEAN; +import static org.apache.avro.Schema.Type.BYTES; +import static org.apache.avro.Schema.Type.DOUBLE; +import static org.apache.avro.Schema.Type.FIXED; +import static org.apache.avro.Schema.Type.FLOAT; +import static org.apache.avro.Schema.Type.INT; +import static org.apache.avro.Schema.Type.LONG; +import static org.apache.avro.Schema.Type.NULL; +import static org.apache.avro.Schema.Type.STRING; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Hashtable; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.avro.Schema; +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.serde2.typeinfo.ITypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; + +/** + * Convert an Avro Schema to a Hive TypeInfo + */ +@LimitedPrivate("Hive") +public abstract class SchemaToTypeInfo { + // Conversion of Avro primitive types to Hive primitive types + // Avro Hive + // Null + // boolean boolean check + // int int check + // long bigint check + // float double check + // double double check + // bytes binary check + // fixed binary check + // string string check + // tinyint + // smallint + + // Map of Avro's primitive types to Hives (for those that are supported by both) + private final Map primitiveTypeToTypeInfo; + private final ITypeInfoFactory typeInfoFactory; + SchemaToTypeInfo(ITypeInfoFactory typeInfoFactory) { + this.typeInfoFactory = typeInfoFactory; + primitiveTypeToTypeInfo = initTypeMap(); + } + + private Map initTypeMap() { + Map theMap = new Hashtable(); + theMap.put(NULL, typeInfoFactory.getPrimitiveTypeInfo("void")); + theMap.put(BOOLEAN, typeInfoFactory.getPrimitiveTypeInfo("boolean")); + theMap.put(INT, typeInfoFactory.getPrimitiveTypeInfo("int")); + theMap.put(LONG, typeInfoFactory.getPrimitiveTypeInfo("bigint")); + theMap.put(FLOAT, typeInfoFactory.getPrimitiveTypeInfo("float")); + theMap.put(DOUBLE, typeInfoFactory.getPrimitiveTypeInfo("double")); + theMap.put(BYTES, typeInfoFactory.getPrimitiveTypeInfo("binary")); + theMap.put(FIXED, typeInfoFactory.getPrimitiveTypeInfo("binary")); + theMap.put(STRING, typeInfoFactory.getPrimitiveTypeInfo("string")); + return Collections.unmodifiableMap(theMap); + } + + /** + * Generate a list of of TypeInfos from an Avro schema. This method is + * currently public due to some weirdness in deserializing unions, but + * will be made private once that is resolved. + * @param schema Schema to generate field types for + * @return List of TypeInfos, each element of which is a TypeInfo derived + * from the schema. + * @throws AvroSerdeException for problems during conversion. + */ + public List generateColumnTypes(Schema schema) throws AvroSerdeException { + return generateColumnTypes (schema, null); + } + + /** + * Generate a list of of TypeInfos from an Avro schema. This method is + * currently public due to some weirdness in deserializing unions, but + * will be made private once that is resolved. + * @param schema Schema to generate field types for + * @param seenSchemas stores schemas processed in the parsing done so far, + * helping to resolve circular references in the schema + * @return List of TypeInfos, each element of which is a TypeInfo derived + * from the schema. + * @throws AvroSerdeException for problems during conversion. + */ + public List generateColumnTypes(Schema schema, + Set seenSchemas) throws AvroSerdeException { + List fields = schema.getFields(); + + List types = new ArrayList(fields.size()); + + for (Schema.Field field : fields) { + types.add(generateTypeInfo(field.schema(), seenSchemas)); + } + + return types; + } + + private InstanceCache typeInfoCache = new InstanceCache() { + @Override + protected TypeInfo makeInstance(Schema s, + Set seenSchemas) + throws AvroSerdeException { + return generateTypeInfoWorker(s, seenSchemas); + } + }; + /** + * Convert an Avro Schema into an equivalent Hive TypeInfo. + * @param schema to record. Must be of record type. + * @param seenSchemas stores schemas processed in the parsing done so far, + * helping to resolve circular references in the schema + * @return TypeInfo matching the Avro schema + * @throws AvroSerdeException for any problems during conversion. + */ + public TypeInfo generateTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + // For bytes type, it can be mapped to decimal. + Schema.Type type = schema.getType(); + if (type == BYTES && AvroSerDeConstants.DECIMAL_TYPE_NAME + .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int precision = 0; + int scale = 0; + try { + precision = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_PRECISION).getIntValue(); + scale = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_SCALE).getIntValue(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain scale value from file schema: " + schema, ex); + } + + try { + MetastoreTypeInfoUtils.validateDecimalParameters(precision, scale); + } catch (Exception ex) { + throw new AvroSerdeException("Invalid precision or scale for decimal type", ex); + } + + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.DECIMAL_TYPE_NAME, precision, scale); + } + + if (type == STRING && AvroSerDeConstants.CHAR_TYPE_NAME + .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int maxLength = 0; + try { + maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); + } + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.CHAR_TYPE_NAME, maxLength); + } + + if (type == STRING && AvroSerDeConstants.VARCHAR_TYPE_NAME + .equalsIgnoreCase(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + int maxLength = 0; + try { + maxLength = schema.getJsonProp(AvroSerDeConstants.AVRO_PROP_MAX_LENGTH).getValueAsInt(); + } catch (Exception ex) { + throw new AvroSerdeException("Failed to obtain maxLength value from file schema: " + schema, ex); + } + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.VARCHAR_TYPE_NAME, maxLength); + } + + if (type == INT && + AvroSerDeConstants.DATE_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.DATE_TYPE_NAME); + } + + if (type == LONG && + AvroSerDeConstants.AVRO_TIMESTAMP_TYPE_NAME.equals(schema.getProp(AvroSerDeConstants.AVRO_PROP_LOGICAL_TYPE))) { + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.TIMESTAMP_TYPE_NAME); + } + + return typeInfoCache.retrieve(schema, seenSchemas); + } + + private TypeInfo generateTypeInfoWorker(Schema schema, + Set seenSchemas) throws AvroSerdeException { + // Avro requires NULLable types to be defined as unions of some type TypeInfo + // and NULL. This is annoying and we're going to hide it from the user. + if(AvroSchemaUtils.isNullableType(schema)) { + return generateTypeInfo( + AvroSchemaUtils.getOtherTypeFromNullableType(schema), seenSchemas); + } + + Schema.Type type = schema.getType(); + if(primitiveTypeToTypeInfo.containsKey(type)) { + return primitiveTypeToTypeInfo.get(type); + } + + switch(type) { + case RECORD: return generateRecordTypeInfo(schema, seenSchemas); + case MAP: return generateMapTypeInfo(schema, seenSchemas); + case ARRAY: return generateArrayTypeInfo(schema, seenSchemas); + case UNION: return generateUnionTypeInfo(schema, seenSchemas); + case ENUM: return generateEnumTypeInfo(schema); + default: throw new AvroSerdeException("Do not yet support: " + schema); + } + } + + private TypeInfo generateRecordTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.RECORD); + + if (seenSchemas == null) { + seenSchemas = Collections.newSetFromMap(new IdentityHashMap()); + } else if (seenSchemas.contains(schema)) { + throw new AvroSerdeException( + "Recursive schemas are not supported. Recursive schema was " + schema + .getFullName()); + } + seenSchemas.add(schema); + + List fields = schema.getFields(); + List fieldNames = new ArrayList(fields.size()); + List typeInfos = new ArrayList(fields.size()); + + for(int i = 0; i < fields.size(); i++) { + fieldNames.add(i, fields.get(i).name()); + typeInfos.add(i, generateTypeInfo(fields.get(i).schema(), seenSchemas)); + } + + return typeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); + } + + /** + * Generate a TypeInfo for an Avro Map. This is made slightly simpler in that + * Avro only allows maps with strings for keys. + */ + private TypeInfo generateMapTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.MAP); + Schema valueType = schema.getValueType(); + TypeInfo ti = generateTypeInfo(valueType, seenSchemas); + + return typeInfoFactory + .getMapTypeInfo(typeInfoFactory.getPrimitiveTypeInfo(ColumnType.STRING_TYPE_NAME), ti); + } + + private TypeInfo generateArrayTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.ARRAY); + Schema itemsType = schema.getElementType(); + TypeInfo itemsTypeInfo = generateTypeInfo(itemsType, seenSchemas); + + return typeInfoFactory.getListTypeInfo(itemsTypeInfo); + } + + private TypeInfo generateUnionTypeInfo(Schema schema, + Set seenSchemas) throws AvroSerdeException { + assert schema.getType().equals(Schema.Type.UNION); + List types = schema.getTypes(); + + + List typeInfos = new ArrayList(types.size()); + + for(Schema type : types) { + typeInfos.add(generateTypeInfo(type, seenSchemas)); + } + + return typeInfoFactory.getUnionTypeInfo(typeInfos); + } + + // Hive doesn't have an Enum type, so we're going to treat them as Strings. + // During the deserialize/serialize stage we'll check for enumness and + // convert as such. + private TypeInfo generateEnumTypeInfo(Schema schema) { + assert schema.getType().equals(Schema.Type.ENUM); + + return typeInfoFactory.getPrimitiveTypeInfo("string"); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java new file mode 100644 index 0000000000000000000000000000000000000000..3328caf7c304f98f4c9bffcd6556ff6c85a4c93f --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/avro/TypeInfoToSchema.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import org.apache.avro.Schema; +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MetastorePrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MetastoreTypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.node.JsonNodeFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Convert Hive TypeInfo to an Avro Schema + */ +public class TypeInfoToSchema { + + private long recordCounter = 0; + + /** + * Converts Hive schema to avro schema + * + * @param columnNames Names of the hive columns + * @param columnTypes Hive Column types + * @param namespace Namespace of Avro schema + * @param name Avro schema name + * @param doc Avro schema doc + * @return Avro Schema + */ + public Schema convert(List columnNames, List columnTypes, + List columnComments, String namespace, String name, String doc) { + + List fields = new ArrayList(); + for (int i = 0; i < columnNames.size(); ++i) { + final String comment = columnComments.size() > i ? columnComments.get(i) : null; + final Schema.Field avroField = createAvroField(columnNames.get(i), columnTypes.get(i), + comment); + fields.addAll(getFields(avroField)); + } + + if (name == null || name.isEmpty()) { + name = "baseRecord"; + } + + Schema avroSchema = Schema.createRecord(name, doc, namespace, false); + avroSchema.setFields(fields); + return avroSchema; + } + + private Schema.Field createAvroField(String name, TypeInfo typeInfo, String comment) { + return new Schema.Field(name, createAvroSchema(typeInfo), comment, null); + } + + private Schema createAvroSchema(TypeInfo typeInfo) { + Schema schema = null; + switch (typeInfo.getCategory()) { + case PRIMITIVE: + schema = createAvroPrimitive(typeInfo); + break; + case LIST: + schema = createAvroArray(typeInfo); + break; + case MAP: + schema = createAvroMap(typeInfo); + break; + case STRUCT: + schema = createAvroRecord(typeInfo); + break; + case UNION: + schema = createAvroUnion(typeInfo); + break; + } + + return wrapInUnionWithNull(schema); + } + + private Schema createAvroPrimitive(TypeInfo typeInfo) { + Schema schema; + String baseTypeName = MetastoreTypeInfoUtils.getBaseName(typeInfo.getTypeName()); + switch (baseTypeName) { + case ColumnType.STRING_TYPE_NAME: + schema = Schema.create(Schema.Type.STRING); + break; + case ColumnType.CHAR_TYPE_NAME: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.CHAR_TYPE_NAME + "\"," + + "\"maxLength\":" + ((MetastorePrimitiveTypeInfo) typeInfo).getParameters()[0] + "}"); + break; + case ColumnType.VARCHAR_TYPE_NAME: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_STRING_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.VARCHAR_TYPE_NAME + "\"," + + "\"maxLength\":" + ((MetastorePrimitiveTypeInfo) typeInfo).getParameters()[0] + "}"); + break; + case ColumnType.BINARY_TYPE_NAME: + schema = Schema.create(Schema.Type.BYTES); + break; + case ColumnType.TINYINT_TYPE_NAME: + schema = Schema.create(Schema.Type.INT); + break; + case ColumnType.SMALLINT_TYPE_NAME: + schema = Schema.create(Schema.Type.INT); + break; + case ColumnType.INT_TYPE_NAME: + schema = Schema.create(Schema.Type.INT); + break; + case ColumnType.BIGINT_TYPE_NAME: + schema = Schema.create(Schema.Type.LONG); + break; + case ColumnType.FLOAT_TYPE_NAME: + schema = Schema.create(Schema.Type.FLOAT); + break; + case ColumnType.DOUBLE_TYPE_NAME: + schema = Schema.create(Schema.Type.DOUBLE); + break; + case ColumnType.BOOLEAN_TYPE_NAME: + schema = Schema.create(Schema.Type.BOOLEAN); + break; + case ColumnType.DECIMAL_TYPE_NAME: + String precision = String.valueOf(((MetastorePrimitiveTypeInfo) typeInfo).getParameters()[0]); + String scale = String.valueOf(((MetastorePrimitiveTypeInfo) typeInfo).getParameters()[1]); + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"bytes\"," + + "\"logicalType\":\"decimal\"," + + "\"precision\":" + precision + "," + + "\"scale\":" + scale + "}"); + break; + case ColumnType.DATE_TYPE_NAME: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_INT_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.DATE_TYPE_NAME + "\"}"); + break; + case ColumnType.TIMESTAMP_TYPE_NAME: + schema = AvroSchemaUtils.getSchemaFor("{" + + "\"type\":\"" + AvroSerDeConstants.AVRO_LONG_TYPE_NAME + "\"," + + "\"logicalType\":\"" + AvroSerDeConstants.AVRO_TIMESTAMP_TYPE_NAME + "\"}"); + break; + case ColumnType.VOID_TYPE_NAME: + schema = Schema.create(Schema.Type.NULL); + break; + default: + throw new UnsupportedOperationException(typeInfo + " is not supported."); + } + return schema; + } + + private Schema createAvroUnion(TypeInfo typeInfo) { + List childSchemas = new ArrayList(); + for (TypeInfo childTypeInfo : ((UnionTypeInfo) typeInfo).getAllUnionObjectTypeInfos()) { + final Schema childSchema = createAvroSchema(childTypeInfo); + if (childSchema.getType() == Schema.Type.UNION) { + childSchemas.addAll(childSchema.getTypes()); + } else { + childSchemas.add(childSchema); + } + } + + return Schema.createUnion(removeDuplicateNullSchemas(childSchemas)); + } + + private Schema createAvroRecord(TypeInfo typeInfo) { + List childFields = new ArrayList(); + final List allStructFieldNames = ((StructTypeInfo) typeInfo).getAllStructFieldNames(); + final List allStructFieldTypeInfos = + ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos(); + if (allStructFieldNames.size() != allStructFieldTypeInfos.size()) { + throw new IllegalArgumentException("Failed to generate avro schema from hive schema. " + + "name and column type differs. names = " + allStructFieldNames + ", types = " + + allStructFieldTypeInfos); + } + + for (int i = 0; i < allStructFieldNames.size(); ++i) { + final TypeInfo childTypeInfo = allStructFieldTypeInfos.get(i); + final Schema.Field grandChildSchemaField = createAvroField(allStructFieldNames.get(i), + childTypeInfo, childTypeInfo.toString()); + final List grandChildFields = getFields(grandChildSchemaField); + childFields.addAll(grandChildFields); + } + + Schema recordSchema = Schema.createRecord("record_" + recordCounter, typeInfo.toString(), + null, false); + ++recordCounter; + recordSchema.setFields(childFields); + return recordSchema; + } + + private Schema createAvroMap(TypeInfo typeInfo) { + TypeInfo keyTypeInfo = ((MapTypeInfo) typeInfo).getMapKeyTypeInfo(); + if (!ColumnType.STRING_TYPE_NAME.equals(keyTypeInfo.getTypeName())) { + throw new UnsupportedOperationException("Key of Map can only be a String"); + } + + TypeInfo valueTypeInfo = ((MapTypeInfo) typeInfo).getMapValueTypeInfo(); + Schema valueSchema = createAvroSchema(valueTypeInfo); + + return Schema.createMap(valueSchema); + } + + private Schema createAvroArray(TypeInfo typeInfo) { + ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo; + Schema listSchema = createAvroSchema(listTypeInfo.getListElementTypeInfo()); + return Schema.createArray(listSchema); + } + + private List getFields(Schema.Field schemaField) { + List fields = new ArrayList(); + + JsonNode nullDefault = JsonNodeFactory.instance.nullNode(); + if (schemaField.schema().getType() == Schema.Type.RECORD) { + for (Schema.Field field : schemaField.schema().getFields()) { + fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), nullDefault)); + } + } else { + fields.add(new Schema.Field(schemaField.name(), schemaField.schema(), schemaField.doc(), + nullDefault)); + } + + return fields; + } + + private Schema wrapInUnionWithNull(Schema schema) { + Schema wrappedSchema = schema; + switch (schema.getType()) { + case NULL: + break; + case UNION: + List existingSchemas = removeDuplicateNullSchemas(schema.getTypes()); + wrappedSchema = Schema.createUnion(existingSchemas); + break; + default: + wrappedSchema = Schema.createUnion(Arrays.asList(Schema.create(Schema.Type.NULL), schema)); + } + + return wrappedSchema; + } + + private List removeDuplicateNullSchemas(List childSchemas) { + List prunedSchemas = new ArrayList(); + boolean isNullPresent = false; + for (Schema schema : childSchemas) { + if (schema.getType() == Schema.Type.NULL) { + isNullPresent = true; + } else { + prunedSchemas.add(schema); + } + } + if (isNullPresent) { + prunedSchemas.add(0, Schema.create(Schema.Type.NULL)); + } + + return prunedSchemas; + } +} \ No newline at end of file diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ITypeInfoFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ITypeInfoFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..24edf70f3098cb38652101f54c7e09243a9dd6e8 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ITypeInfoFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; + +import java.util.List; + +@LimitedPrivate("Hive") +public interface ITypeInfoFactory { + /** + * Get or create a Primitive TypeInfo object of name typeName and parameters provided by + * paramaters. Eg. a primitive typeInfo of char(10) can be represented a typename --> char + * and 10 as the parameter. Similarly, decimal(10,2) has typename decimal and 10,2 as + * parameters + * + * @param typeName name of the type + * @param parameters optional parameters in case of parameterized primitive types + * @return TypeInfo representing the primitive typeInfo + */ + MetastorePrimitiveTypeInfo getPrimitiveTypeInfo(String typeName, Object... parameters); + + /** + * Get or create a Map type TypeInfo + * + * @param keyTypeInfo TypeInfo for the key + * @param valueTypeInfo TypeInfo for the value + * @return MapTypeInfo + */ + MapTypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo); + + /** + * Get or create a List type TypeInfo + * + * @param listElementTypeInfo TypeInfo of the list elements + * @return ListTypeInfo + */ + ListTypeInfo getListTypeInfo(TypeInfo listElementTypeInfo); + + /** + * Get or create a UnionTypeInfo + * + * @param typeInfos child TypeInfos for the UnionTypeInfo + * @return UnionTypeInfo + */ + UnionTypeInfo getUnionTypeInfo(List typeInfos); + + /** + * Get or create a StructTypeInfo + * + * @param names names of the fields in the struct typeInfo + * @param typeInfos TypeInfos for each fields + * @return StructTypeInfo + */ + StructTypeInfo getStructTypeInfo(List names, List typeInfos); +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..d86302818d0a5a13cdc3eba0292ed9c29230888c --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/ListTypeInfo.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.ColumnType; + +import java.io.Serializable; + + +/** + * A List Type has homogeneous elements. All elements of the List has the same + * TypeInfo which is returned by getListElementTypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class ListTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + private TypeInfo listElementTypeInfo; + + /** + * For java serialization use only. + */ + public ListTypeInfo() { + } + + @Override + public String getTypeName() { + return ColumnType.LIST_TYPE_NAME + "<" + + listElementTypeInfo.getTypeName() + ">"; + } + + /** + * For java serialization use only. + */ + public void setListElementTypeInfo(TypeInfo listElementTypeInfo) { + this.listElementTypeInfo = listElementTypeInfo; + } + + /** + * For TypeInfoFactory use only. + */ + ListTypeInfo(TypeInfo elementTypeInfo) { + listElementTypeInfo = elementTypeInfo; + } + + @Override + public Category getCategory() { + return Category.LIST; + } + + public TypeInfo getListElementTypeInfo() { + return listElementTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ListTypeInfo)) { + return false; + } + return getListElementTypeInfo().equals( + ((ListTypeInfo) other).getListElementTypeInfo()); + } + + @Override + public int hashCode() { + return listElementTypeInfo.hashCode(); + } + +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..9ed9d5e858e3977c8e506c5b58de9f07b7162dc9 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MapTypeInfo.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.ColumnType; + +import java.io.Serializable; + + +/** + * A Map Type has homogeneous keys and homogeneous values. All keys of the Map + * have the same TypeInfo, which is returned by getMapKeyTypeInfo(); and all + * values of the Map has the same TypeInfo, which is returned by + * getMapValueTypeInfo(). + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class MapTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private TypeInfo mapKeyTypeInfo; + private TypeInfo mapValueTypeInfo; + + /** + * For java serialization use only. + */ + public MapTypeInfo() { + } + + @Override + public String getTypeName() { + return ColumnType.MAP_TYPE_NAME + "<" + + mapKeyTypeInfo.getTypeName() + "," + mapValueTypeInfo.getTypeName() + + ">"; + } + + /** + * For java serialization use only. + */ + public void setMapKeyTypeInfo(TypeInfo mapKeyTypeInfo) { + this.mapKeyTypeInfo = mapKeyTypeInfo; + } + + /** + * For java serialization use only. + */ + public void setMapValueTypeInfo(TypeInfo mapValueTypeInfo) { + this.mapValueTypeInfo = mapValueTypeInfo; + } + + // For TypeInfoFactory use only + MapTypeInfo(TypeInfo keyTypeInfo, TypeInfo valueTypeInfo) { + mapKeyTypeInfo = keyTypeInfo; + mapValueTypeInfo = valueTypeInfo; + } + + @Override + public Category getCategory() { + return Category.MAP; + } + + public TypeInfo getMapKeyTypeInfo() { + return mapKeyTypeInfo; + } + + public TypeInfo getMapValueTypeInfo() { + return mapValueTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof MapTypeInfo)) { + return false; + } + MapTypeInfo o = (MapTypeInfo) other; + return o.getMapKeyTypeInfo().equals(getMapKeyTypeInfo()) + && o.getMapValueTypeInfo().equals(getMapValueTypeInfo()); + } + + @Override + public int hashCode() { + return mapKeyTypeInfo.hashCode() ^ mapValueTypeInfo.hashCode(); + } + +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastorePrimitiveTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastorePrimitiveTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..38c0e2b5987713ce38e98a05da78b55eac075d64 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastorePrimitiveTypeInfo.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; + +import java.io.Serializable; + +/** + * This class represents a PrimitiveTypeInfo. Hive extends this class to create PrimitiveTypeInfo + */ +@LimitedPrivate("Hive") +public class MetastorePrimitiveTypeInfo extends TypeInfo implements Serializable { + // Base name (varchar vs fully qualified name such as varchar(200)). + protected String typeName; + + public MetastorePrimitiveTypeInfo() { + } + + public MetastorePrimitiveTypeInfo(String typeName) { + this.typeName = typeName; + } + + /** + * Returns the category of this TypeInfo. + */ + @Override + public Category getCategory() { + return Category.PRIMITIVE; + } + + // The following 2 methods are for java serialization use only. + public void setTypeName(String typeName) { + this.typeName = typeName; + } + + @Override + public String getTypeName() { + return typeName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + MetastorePrimitiveTypeInfo pti = (MetastorePrimitiveTypeInfo) other; + + return this.typeName.equals(pti.typeName); + } + + /** + * Generate the hashCode for this TypeInfo. + */ + @Override + public int hashCode() { + return typeName.hashCode(); + } + + @Override + public String toString() { + return typeName; + } + + private static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + + /** + * parameterized TypeInfos should override this to return array of parameters + * @return + */ + public Object[] getParameters() { + //default is no parameters + return EMPTY_OBJECT_ARRAY; + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..de10641b164f3c56fa4b67f89e0bc28bdad29ef3 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoFactory.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.hive.metastore.ColumnType; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class MetastoreTypeInfoFactory implements ITypeInfoFactory { + + private static final MetastoreTypeInfoFactory instance = new MetastoreTypeInfoFactory(); + + public static final MetastoreTypeInfoFactory getInstance() { + return instance; + } + private static ConcurrentHashMap cachedPrimitiveTypeInfo = + new ConcurrentHashMap<>(); + + @Override + public MetastorePrimitiveTypeInfo getPrimitiveTypeInfo(String typeName, Object... parameters) { + String qualifiedTypeName = MetastoreTypeInfoUtils + .getQualifiedPrimitiveTypeName(typeName, parameters); + MetastorePrimitiveTypeInfo result = cachedPrimitiveTypeInfo.get(qualifiedTypeName); + if (result != null) { + return result; + } + + if (ColumnType.CHAR_TYPE_NAME.equals(typeName) || ColumnType.VARCHAR_TYPE_NAME + .equals(typeName)) { + MetastoreTypeInfoUtils.validateCharVarCharParameters((int) parameters[0]); + } else if (ColumnType.DECIMAL_TYPE_NAME.equals(typeName)) { + MetastoreTypeInfoUtils.validateDecimalParameters((int) parameters[0], (int) parameters[1]); + } + // Not found in the cache. Must be parameterized types. Create it. + result = new MetastorePrimitiveTypeInfo(qualifiedTypeName); + + MetastorePrimitiveTypeInfo prev = cachedPrimitiveTypeInfo.putIfAbsent(qualifiedTypeName, result); + if (prev != null) { + result = prev; + } + return result; + } + + private static ConcurrentHashMap, MapTypeInfo> cachedMapTypeInfo = + new ConcurrentHashMap<>(); + @Override + public MapTypeInfo getMapTypeInfo(TypeInfo keyTypeInfo, + TypeInfo valueTypeInfo) { + ArrayList signature = new ArrayList(2); + signature.add(keyTypeInfo); + signature.add(valueTypeInfo); + MapTypeInfo result = cachedMapTypeInfo.get(signature); + if (result == null) { + result = new MapTypeInfo(keyTypeInfo, valueTypeInfo); + MapTypeInfo prev = cachedMapTypeInfo.putIfAbsent(signature, result); + if (prev != null) { + result = prev; + } + } + return result; + } + + private static ConcurrentHashMap cachedListTypeInfo = new ConcurrentHashMap<>(); + + @Override + public ListTypeInfo getListTypeInfo(TypeInfo listElementTypeInfo) { + ListTypeInfo result = cachedListTypeInfo.get(listElementTypeInfo); + if (result == null) { + result = new ListTypeInfo(listElementTypeInfo); + ListTypeInfo prev = cachedListTypeInfo.putIfAbsent(listElementTypeInfo, result); + if (prev != null) { + result = prev; + } + } + return result; + } + + private static ConcurrentHashMap, UnionTypeInfo> cachedUnionTypeInfo = + new ConcurrentHashMap<>(); + + @Override + public UnionTypeInfo getUnionTypeInfo(List typeInfos) { + UnionTypeInfo result = cachedUnionTypeInfo.get(typeInfos); + if (result == null) { + result = new UnionTypeInfo(typeInfos); + UnionTypeInfo prev = cachedUnionTypeInfo.putIfAbsent(typeInfos, result); + if (prev != null) { + result = prev; + } + } + return result; + } + static ConcurrentHashMap>, StructTypeInfo> cachedStructTypeInfo = + new ConcurrentHashMap<>(); + @Override + public StructTypeInfo getStructTypeInfo(List names, + List typeInfos) { + ArrayList> signature = new ArrayList>(2); + signature.add(names); + signature.add(typeInfos); + StructTypeInfo result = cachedStructTypeInfo.get(signature); + if (result == null) { + result = new StructTypeInfo(names, typeInfos); + StructTypeInfo prev = cachedStructTypeInfo.putIfAbsent(signature, result); + if (prev != null) { + result = prev; + } + } + return result; + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..780dc50be6d71cf73979352f26843f4f4e7c79dc --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/MetastoreTypeInfoUtils.java @@ -0,0 +1,59 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +public class MetastoreTypeInfoUtils { + private MetastoreTypeInfoUtils() { + } + + /** + * Metastore is not supposed to enforce type ranges. The type range checks should be left + * to the implementation engines. This method does a very lenient check which is obvious + * and makes sense for overall sanity of decimal types + * @param precision decimal precision value + * @param scale decimal scale value + */ + public static void validateDecimalParameters(int precision, int scale) { + if (precision < 0) { + throw new IllegalArgumentException("Precision cannot be negative"); + } + if (scale < 0) { + throw new IllegalArgumentException("Scale cannot be negative"); + } + } + + /** + * Metastore is not supposed to enforce type ranges. The type range checks should be left + * to the implementation engines. This method does a very lenient check which is obvious + * and makes sense for overall sanity of char types + * @param length + */ + public static void validateCharVarCharParameters(int length) { + if (length < 0) { + throw new IllegalArgumentException("Length cannot be negative"); + } + } + + static String getQualifiedPrimitiveTypeName(String type, Object... parameters) { + StringBuilder sb = new StringBuilder(type); + if (parameters == null || parameters.length == 0) { + return sb.toString(); + } + sb.append('('); + for (int i = 0; i < parameters.length; i++) { + sb.append(parameters[i]); + if (i != (parameters.length - 1)) { + sb.append(','); + } + } + sb.append(')'); + return sb.toString(); + } + + public static String getBaseName(String typeName) { + int idx = typeName.indexOf('('); + if (idx == -1) { + return typeName; + } else { + return typeName.substring(0, idx); + } + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..eed1141bcd28a7489bd48c99e353880ae6d951d0 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/StructTypeInfo.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.ColumnType; + +/** + * StructTypeInfo represents the TypeInfo of a struct. A struct contains one or + * more fields each of which has a unique name and its own TypeInfo. Different + * fields can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public final class StructTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private ArrayList allStructFieldNames; + private ArrayList allStructFieldTypeInfos; + + /** + * For java serialization use only. + */ + public StructTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(ColumnType.STRUCT_TYPE_NAME + "<"); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allStructFieldNames.get(i)); + sb.append(":"); + sb.append(allStructFieldTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldNames(ArrayList allStructFieldNames) { + this.allStructFieldNames = allStructFieldNames; + } + + /** + * For java serialization use only. + */ + public void setAllStructFieldTypeInfos( + ArrayList allStructFieldTypeInfos) { + this.allStructFieldTypeInfos = allStructFieldTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + StructTypeInfo(List names, List typeInfos) { + allStructFieldNames = new ArrayList(names); + allStructFieldTypeInfos = new ArrayList(typeInfos); + } + + @Override + public Category getCategory() { + return Category.STRUCT; + } + + public ArrayList getAllStructFieldNames() { + return allStructFieldNames; + } + + public ArrayList getAllStructFieldTypeInfos() { + return allStructFieldTypeInfos; + } + + public TypeInfo getStructFieldTypeInfo(String field) { + String fieldLowerCase = field.toLowerCase(); + for (int i = 0; i < allStructFieldNames.size(); i++) { + if (fieldLowerCase.equalsIgnoreCase(allStructFieldNames.get(i))) { + return allStructFieldTypeInfos.get(i); + } + } + throw new RuntimeException("cannot find field " + field + + "(lowercase form: " + fieldLowerCase + ") in " + allStructFieldNames); + // return null; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof StructTypeInfo)) { + return false; + } + StructTypeInfo o = (StructTypeInfo) other; + Iterator namesIterator = getAllStructFieldNames().iterator(); + Iterator otherNamesIterator = o.getAllStructFieldNames().iterator(); + + // Compare the field names using ignore-case semantics + while (namesIterator.hasNext() && otherNamesIterator.hasNext()) { + if (!namesIterator.next().equalsIgnoreCase(otherNamesIterator.next())) { + return false; + } + } + + // Different number of field names + if (namesIterator.hasNext() || otherNamesIterator.hasNext()) { + return false; + } + + // Compare the field types + return o.getAllStructFieldTypeInfos().equals(getAllStructFieldTypeInfos()); + } + + @Override + public int hashCode() { + return allStructFieldNames.hashCode() ^ allStructFieldTypeInfos.hashCode(); + } + +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..c5ed68ccba926d42ccde1e1b42eafbf244575b00 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfo.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.serde2.typeinfo; + +import java.io.Serializable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Stores information about a type. Always use the TypeInfoFactory to create new + * TypeInfo objects. + * + * We support 8 categories of types: + * 1. Primitive objects (String, Number, etc) + * 2. List objects (a list of objects of a single type) + * 3. Map objects (a map from objects of one type to objects of another type) + * 4. Struct objects (a list of fields with names and their own types) + * 5. Union objects + * 6. Decimal objects + * 7. Char objects + * 8. Varchar objects + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + protected TypeInfo() { + } + + /** + * Category. + * + */ + public static enum Category { + PRIMITIVE, LIST, MAP, STRUCT, UNION + } + + /** + * The Category of this TypeInfo. Possible values are Primitive, List, Map, + * Struct and Union, which corresponds to the 5 sub-classes of TypeInfo. + */ + public abstract Category getCategory(); + + /** + * A String representation of the TypeInfo. + */ + public abstract String getTypeName(); + + /** + * String representing the qualified type name. + * Qualified types should override this method. + * @return + */ + public String getQualifiedName() { + return getTypeName(); + } + + @Override + public String toString() { + return getTypeName(); + } + + @Override + public abstract boolean equals(Object o); + + @Override + public abstract int hashCode(); + + public boolean accept(TypeInfo other) { + return this.equals(other); + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java new file mode 100644 index 0000000000000000000000000000000000000000..b6e07d9260c359ba5c7d77ded3cc020b2b3b1ce2 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoParser.java @@ -0,0 +1,326 @@ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.metastore.ColumnType; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * Parse a recursive TypeInfo list String. For example, the following inputs + * are valid inputs: + * "int,string,map,list>>,list>" + * The separators between TypeInfos can be ",", ":", or ";". + * + * In order to use this class: TypeInfoParser parser = new + * TypeInfoParser("int,string"); ArrayList typeInfos = + * parser.parseTypeInfos(); + */ +@LimitedPrivate("Hive") +public class TypeInfoParser { + + private static class Token { + public int position; + public String text; + public boolean isType; + + @Override + public String toString() { + return "" + position + ":" + text; + } + }; + + private static boolean isTypeChar(char c) { + return Character.isLetterOrDigit(c) || c == '_' || c == '.' || c == ' ' || c == '$'; + } + + /** + * Tokenize the typeInfoString. The rule is simple: all consecutive + * alphadigits and '_', '.' are in one token, and all other characters are + * one character per token. + * + * tokenize("map") should return + * ["map","<","int",",","string",">"] + * + * Note that we add '$' in new Calcite return path. As '$' will not appear + * in any type in Hive, it is safe to do so. + */ + private static ArrayList tokenize(String typeInfoString) { + ArrayList tokens = new ArrayList(0); + int begin = 0; + int end = 1; + while (end <= typeInfoString.length()) { + // last character ends a token? + // if there are quotes, all the text between the quotes + // is considered a single token (this can happen for + // timestamp with local time-zone) + if (begin > 0 && + typeInfoString.charAt(begin - 1) == '(' && + typeInfoString.charAt(begin) == '\'') { + // Ignore starting quote + begin++; + do { + end++; + } while (typeInfoString.charAt(end) != '\''); + } else if (typeInfoString.charAt(begin) == '\'' && + typeInfoString.charAt(begin + 1) == ')') { + // Ignore closing quote + begin++; + end++; + } + if (end == typeInfoString.length() + || !isTypeChar(typeInfoString.charAt(end - 1)) + || !isTypeChar(typeInfoString.charAt(end))) { + Token t = new Token(); + t.position = begin; + t.text = typeInfoString.substring(begin, end); + t.isType = isTypeChar(typeInfoString.charAt(begin)); + tokens.add(t); + begin = end; + } + end++; + } + return tokens; + } + + public TypeInfoParser(String typeInfoString, ITypeInfoFactory typeInfoFactory) { + this.typeInfoString = typeInfoString; + this.typeInfoFactory = typeInfoFactory; + typeInfoTokens = tokenize(typeInfoString); + } + + private final String typeInfoString; + private final ArrayList typeInfoTokens; + private final ITypeInfoFactory typeInfoFactory; + private ArrayList typeInfos; + private int iToken; + + public ArrayList parseTypeInfos() { + typeInfos = new ArrayList(); + iToken = 0; + while (iToken < typeInfoTokens.size()) { + typeInfos.add(parseType()); + if (iToken < typeInfoTokens.size()) { + Token separator = typeInfoTokens.get(iToken); + if (",".equals(separator.text) || ";".equals(separator.text) + || ":".equals(separator.text)) { + iToken++; + } else { + throw new IllegalArgumentException( + "Error: ',', ':', or ';' expected at position " + + separator.position + " from '" + typeInfoString + "' " + + typeInfoTokens); + } + } + } + return typeInfos; + } + + private Token peek() { + if (iToken < typeInfoTokens.size()) { + return typeInfoTokens.get(iToken); + } else { + return null; + } + } + + private Token expect(String item) { + return expect(item, null); + } + + private Token expect(String item, String alternative) { + if (iToken >= typeInfoTokens.size()) { + throw new IllegalArgumentException("Error: " + item + + " expected at the end of '" + typeInfoString + "'"); + } + Token t = typeInfoTokens.get(iToken); + if (item.equals("type")) { + if (!ColumnType.LIST_TYPE_NAME.equals(t.text) + && !ColumnType.MAP_TYPE_NAME.equals(t.text) + && !ColumnType.STRUCT_TYPE_NAME.equals(t.text) + && !ColumnType.UNION_TYPE_NAME.equals(t.text) + && !ColumnType.PrimitiveTypes.contains(t.text) + && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else if (item.equals("name")) { + if (!t.isType && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } else { + if (!item.equals(t.text) && !t.text.equals(alternative)) { + throw new IllegalArgumentException("Error: " + item + + " expected at the position " + t.position + " of '" + + typeInfoString + "' but '" + t.text + "' is found."); + } + } + iToken++; + return t; + } + + private String[] parseParams() { + List params = new LinkedList(); + + Token t = peek(); + if (t != null && t.text.equals("(")) { + expect("("); + + // checking for null in the for-loop condition prevents null-ptr exception + // and allows us to fail more gracefully with a parsing error. + for(t = peek(); (t == null) || !t.text.equals(")"); t = expect(",",")")) { + params.add(expect("name").text); + } + if (params.size() == 0) { + throw new IllegalArgumentException( + "type parameters expected for type string " + typeInfoString); + } + } + + return params.toArray(new String[params.size()]); + } + + private TypeInfo parseType() { + + Token t = expect("type"); + + // Is this a primitive type? + if (ColumnType.PrimitiveTypes.contains(t.text)) { + String[] params = parseParams(); + switch (t.text) { + case ColumnType.CHAR_TYPE_NAME: + case ColumnType.VARCHAR_TYPE_NAME: + if (params == null || params.length == 0) { + throw new IllegalArgumentException(t.text + + " type is specified without length: " + typeInfoString); + } + + int length = 1; + if (params.length == 1) { + length = Integer.parseInt(params[0]); + if (ColumnType.VARCHAR_TYPE_NAME.equals(t.text)) { + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.VARCHAR_TYPE_NAME, length); + } else { + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.CHAR_TYPE_NAME, length); + } + } else if (params.length > 1) { + throw new IllegalArgumentException( + "Type " + t.text + " only takes one parameter, but " + + params.length + " is seen"); + } + + case ColumnType.DECIMAL_TYPE_NAME: + //TODO do we need to support this? this works only by luck because + //standalone-metastore depends on storage-api and HiveDecimal happens to be + //in storage-api + int precision = HiveDecimal.USER_DEFAULT_PRECISION; + int scale = HiveDecimal.USER_DEFAULT_SCALE; + if (params == null || params.length == 0) { + // It's possible that old metadata still refers to "decimal" as a column type w/o + // precision/scale. In this case, the default (10,0) is assumed. Thus, do nothing here. + } else if (params.length == 1) { + // only precision is specified + precision = Integer.valueOf(params[0]); + } else if (params.length == 2) { + // New metadata always have two parameters. + precision = Integer.parseInt(params[0]); + scale = Integer.parseInt(params[1]); + } else if (params.length > 2) { + throw new IllegalArgumentException("Type decimal only takes two parameter, but " + + params.length + " is seen"); + } + return typeInfoFactory.getPrimitiveTypeInfo(ColumnType.DECIMAL_TYPE_NAME, precision, scale); + + default: + return typeInfoFactory.getPrimitiveTypeInfo(t.text); + } + } + + // Is this a list type? + if (ColumnType.LIST_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo listElementType = parseType(); + expect(">"); + return typeInfoFactory.getListTypeInfo(listElementType); + } + + // Is this a map type? + if (ColumnType.MAP_TYPE_NAME.equals(t.text)) { + expect("<"); + TypeInfo mapKeyType = parseType(); + expect(","); + TypeInfo mapValueType = parseType(); + expect(">"); + return typeInfoFactory.getMapTypeInfo(mapKeyType, mapValueType); + } + + // Is this a struct type? + if (ColumnType.STRUCT_TYPE_NAME.equals(t.text)) { + ArrayList fieldNames = new ArrayList<>(); + ArrayList fieldTypeInfos = new ArrayList<>(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of struct + break; + } + } + Token name = expect("name",">"); + if (name.text.equals(">")) { + break; + } + fieldNames.add(name.text); + expect(":"); + fieldTypeInfos.add(parseType()); + } while (true); + + return typeInfoFactory.getStructTypeInfo(fieldNames, fieldTypeInfos); + } + // Is this a union type? + if (ColumnType.UNION_TYPE_NAME.equals(t.text)) { + List objectTypeInfos = new ArrayList<>(); + boolean first = true; + do { + if (first) { + expect("<"); + first = false; + } else { + Token separator = expect(">", ","); + if (separator.text.equals(">")) { + // end of union + break; + } + } + objectTypeInfos.add(parseType()); + } while (true); + + return typeInfoFactory.getUnionTypeInfo(objectTypeInfos); + } + + throw new RuntimeException("Internal error parsing position " + + t.position + " of '" + typeInfoString + "'"); + } + + public PrimitiveParts parsePrimitiveParts() { + PrimitiveParts parts = new PrimitiveParts(); + Token t = expect("type"); + parts.typeName = t.text; + parts.typeParams = parseParams(); + return parts; + } + + public static class PrimitiveParts { + public String typeName; + public String[] typeParams; + } +} diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..6b6d14cf991a43a227367da2e16c059f769bf224 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/serde2/typeinfo/UnionTypeInfo.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.typeinfo; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.ColumnType; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * UnionTypeInfo represents the TypeInfo of an union. A union holds only one + * field of the specified fields at any point of time. The fields, a Union can + * hold, can have the same or different TypeInfo. + * + * Always use the TypeInfoFactory to create new TypeInfo objects, instead of + * directly creating an instance of this class. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class UnionTypeInfo extends TypeInfo implements Serializable { + + private static final long serialVersionUID = 1L; + + private List allUnionObjectTypeInfos; + + /** + * For java serialization use only. + */ + public UnionTypeInfo() { + } + + @Override + public String getTypeName() { + StringBuilder sb = new StringBuilder(); + sb.append(ColumnType.UNION_TYPE_NAME + "<"); + for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(allUnionObjectTypeInfos.get(i).getTypeName()); + } + sb.append(">"); + return sb.toString(); + } + + /** + * For java serialization use only. + */ + public void setAllUnionObjectTypeInfos( + List allUnionObjectTypeInfos) { + this.allUnionObjectTypeInfos = allUnionObjectTypeInfos; + } + + /** + * For TypeInfoFactory use only. + */ + UnionTypeInfo(List typeInfos) { + allUnionObjectTypeInfos = new ArrayList(); + allUnionObjectTypeInfos.addAll(typeInfos); + } + + @Override + public Category getCategory() { + return Category.UNION; + } + + public List getAllUnionObjectTypeInfos() { + return allUnionObjectTypeInfos; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof UnionTypeInfo)) { + return false; + } + UnionTypeInfo o = (UnionTypeInfo) other; + + // Compare the field types + return o.getAllUnionObjectTypeInfos().equals(getAllUnionObjectTypeInfos()); + } + + @Override + public int hashCode() { + return allUnionObjectTypeInfos.hashCode(); + } +} diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java new file mode 100644 index 0000000000000000000000000000000000000000..6ba90d64ef85a6c0bbd6a18e4d42b36bcad3354c --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/schema/reader/TestDefaultStorageSchemaReader.java @@ -0,0 +1,598 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.schema.reader; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.DefaultStorageSchemaReader; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.utils.AvroSchemaUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +@Category(MetastoreUnitTest.class) +public class TestDefaultStorageSchemaReader { + private static final Logger LOG = LoggerFactory.getLogger(TestDefaultStorageSchemaReader.class); + private static final String TEST_DB_NAME = "TEST_DB"; + private static final String TEST_TABLE_NAME = "TEST_TABLE"; + private HiveMetaStoreClient client; + private Configuration conf; + private Warehouse warehouse; + private static final int DEFAULT_LIMIT_PARTITION_REQUEST = 100; + private static final String AVRO_SERIALIZATION_LIB = + "org.apache.hadoop.hive.serde2.avro.AvroSerDe"; + + // These schemata are used in other tests + static public final String MAP_WITH_PRIMITIVE_VALUE_TYPE = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"oneMap\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":\"long\"}\n" + + "\t}\n" + + " ]\n" + + "}"; + static public final String ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"oneArray\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"anArray\",\n" + + " \"type\":{\"type\":\"array\",\n" + + " \"items\":\"string\"}\n" + + "\t}\n" + + " ]\n" + + "}"; + public static final String RECORD_SCHEMA = "{\n" + + " \"namespace\": \"testing.test.mctesty\",\n" + + " \"name\": \"oneRecord\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aRecord\",\n" + + " \"type\":{\"type\":\"record\",\n" + + " \"name\":\"recordWithinARecord\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"int1\",\n" + + " \"type\":\"int\"\n" + + " },\n" + + " {\n" + + " \"name\":\"boolean1\",\n" + + " \"type\":\"boolean\"\n" + + " },\n" + + " {\n" + + " \"name\":\"long1\",\n" + + " \"type\":\"long\"\n" + + " }\n" + + " ]}\n" + + " }\n" + + " ]\n" + + "}"; + public static final String NULLABLE_RECORD_SCHEMA = "[\"null\", " + RECORD_SCHEMA + "]"; + public static final String UNION_SCHEMA = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"int\", \"string\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_2 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"null\", \"int\", \"string\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_3 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"float\",\"int\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String UNION_SCHEMA_4 = "{\n" + + " \"namespace\": \"test.a.rossa\",\n" + + " \"name\": \"oneUnion\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aUnion\",\n" + + " \"type\":[\"int\",\"float\",\"long\"]\n" + + " }\n" + + " ]\n" + + "}"; + public static final String ENUM_SCHEMA = "{\n" + + " \"namespace\": \"clever.namespace.name.in.space\",\n" + + " \"name\": \"oneEnum\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"baddies\",\n" + + " \"type\":{\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " + + "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}\n" + + " \n" + + " \n" + + " }\n" + + " ]\n" + + "}"; + public static final String FIXED_SCHEMA = "{\n" + + " \"namespace\": \"ecapseman\",\n" + + " \"name\": \"oneFixed\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"hash\",\n" + + " \"type\":{\"type\": \"fixed\", \"name\": \"MD5\", \"size\": 16}\n" + + " }\n" + + " ]\n" + + "}"; + public static final String NULLABLE_STRING_SCHEMA = "{\n" + + " \"type\": \"record\", \n" + + " \"name\": \"nullableUnionTest\",\n" + + " \"fields\" : [\n" + + " {\"name\":\"nullableString\", \"type\":[\"null\", \"string\"]}\n" + + " ]\n" + + "}"; + public static final String MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA = "{\n" + + " \"namespace\": \"testing\",\n" + + " \"name\": \"mapWithNullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"aMap\",\n" + + " \"type\":{\"type\":\"map\",\n" + + " \"values\":[\"null\",\"long\"]}\n" + + "\t}\n" + + " ]\n" + + "}"; + public static final String NULLABLE_ENUM_SCHEMA = "{\n" + + " \"namespace\": \"clever.namespace.name.in.space\",\n" + + " \"name\": \"nullableUnionTest\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"nullableEnum\",\n" + + " \"type\": [\"null\", {\"type\":\"enum\",\"name\":\"villians\", \"symbols\": " + + "[\"DALEKS\", \"CYBERMEN\", \"SLITHEEN\", \"JAGRAFESS\"]}]\n" + + " \n" + + " \n" + + " }\n" + + " ]\n" + + "}"; + public static final String BYTES_SCHEMA = "{\n" + + " \"type\": \"record\", \n" + + " \"name\": \"bytesTest\",\n" + + " \"fields\" : [\n" + + " {\"name\":\"bytesField\", \"type\":\"bytes\"}\n" + + " ]\n" + + "}"; + + public static final String KITCHEN_SINK_SCHEMA = "{\n" + + " \"namespace\": \"org.apache.hadoop.hive\",\n" + + " \"name\": \"kitchsink\",\n" + + " \"type\": \"record\",\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\":\"string1\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"string2\",\n" + + " \"type\":\"string\"\n" + + " },\n" + + " {\n" + + " \"name\":\"int1\",\n" + + " \"type\":\"int\"\n" + + " },\n" + + " {\n" + + " \"name\":\"boolean1\",\n" + + " \"type\":\"boolean\"\n" + + " },\n" + + " {\n" + + " \"name\":\"long1\",\n" + + " \"type\":\"long\"\n" + + " },\n" + + " {\n" + + " \"name\":\"float1\",\n" + + " \"type\":\"float\"\n" + + " },\n" + + " {\n" + + " \"name\":\"double1\",\n" + + " \"type\":\"double\"\n" + + " },\n" + + " {\n" + + " \"name\":\"inner_record1\",\n" + + " \"type\":{ \"type\":\"record\",\n" + + " \"name\":\"inner_record1_impl\",\n" + + " \"fields\": [\n" + + " {\"name\":\"int_in_inner_record1\",\n" + + " \"type\":\"int\"},\n" + + " {\"name\":\"string_in_inner_record1\",\n" + + " \"type\":\"string\"}\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\":\"enum1\",\n" + + " \"type\":{\"type\":\"enum\", \"name\":\"enum1_values\", " + + "\"symbols\":[\"ENUM1_VALUES_VALUE1\",\"ENUM1_VALUES_VALUE2\", \"ENUM1_VALUES_VALUE3\"]}\n" + + " },\n" + + " {\n" + + " \"name\":\"array1\",\n" + + " \"type\":{\"type\":\"array\", \"items\":\"string\"}\n" + + " },\n" + + " {\n" + + " \"name\":\"map1\",\n" + + " \"type\":{\"type\":\"map\", \"values\":\"string\"}\n" + + " },\n" + + " {\n" + + " \"name\":\"union1\",\n" + + " \"type\":[\"float\", \"boolean\", \"string\"]\n" + + " },\n" + + " {\n" + + " \"name\":\"fixed1\",\n" + + " \"type\":{\"type\":\"fixed\", \"name\":\"fourbytes\", \"size\":4}\n" + + " },\n" + + " {\n" + + " \"name\":\"null1\",\n" + + " \"type\":\"null\"\n" + + " },\n" + + " {\n" + + " \"name\":\"UnionNullInt\",\n" + + " \"type\":[\"int\", \"null\"]\n" + + " },\n" + + " {\n" + + " \"name\":\"bytes1\",\n" + + " \"type\":\"bytes\"\n" + + " }\n" + + " ]\n" + + "}"; + + @Before + public void setUp() throws Exception { + conf = MetastoreConf.newMetastoreConf(); + warehouse = new Warehouse(conf); + + // set some values to use for getting conf. vars + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.METRICS_ENABLED, true); + conf.set("hive.key1", "value1"); + conf.set("hive.key2", "http://www.example.com"); + conf.set("hive.key3", ""); + conf.set("hive.key4", "0"); + conf.set("datanucleus.autoCreateTables", "false"); + + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX, 2); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.LIMIT_PARTITION_REQUEST, + DEFAULT_LIMIT_PARTITION_REQUEST); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.STORAGE_SCHEMA_READER_IMPL, + DefaultStorageSchemaReader.class.getName()); + client = createClient(); + } + + @After + public void closeClient() { + client.close(); + } + + private void silentDropDatabase(String dbName) throws TException { + try { + for (String tableName : client.getTables(dbName, "*")) { + client.dropTable(dbName, tableName); + } + client.dropDatabase(dbName); + } catch (NoSuchObjectException | InvalidOperationException e) { + // NOP + } + } + + private HiveMetaStoreClient createClient() throws Exception { + try { + return new HiveMetaStoreClient(conf); + } catch (Throwable e) { + System.err.println("Unable to open the metastore"); + System.err.println(StringUtils.stringifyException(e)); + throw new Exception(e); + } + } + + @Test + public void testSimpleAvroTable() throws TException, IOException { + List fields = new ArrayList<>(2); + FieldSchema field = new FieldSchema(); + field.setName("name"); + field.setType("string"); + field.setComment("Test name comment"); + fields.add(field); + + field = new FieldSchema(); + field.setName("age"); + field.setType("int"); + field.setComment("Test age comment"); + fields.add(field); + + createTable(TEST_DB_NAME, TEST_TABLE_NAME, AVRO_SERIALIZATION_LIB, fields, null); + List retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + verifyTableFields(fields, retFields, null); + } + + private Table createTable(String dbName, String tblName, String serializationLib, + List fields, Map tblProperties) throws TException, IOException { + client.dropTable(dbName, tblName); + silentDropDatabase(dbName); + Database db = new Database(); + db.setName(dbName); + client.createDatabase(db); + db = client.getDatabase(dbName); + Path dbPath = new Path(db.getLocationUri()); + FileSystem fs = FileSystem.get(dbPath.toUri(), conf); + String typeName = "dummy"; + client.dropType(typeName); + Type typ1 = new Type(); + typ1.setName(typeName); + typ1.setFields(fields); + client.createType(typ1); + + Table tbl = new TableBuilder().setDbName(dbName).setTableName(tblName).setCols(typ1.getFields()) + .setSerdeLib(serializationLib).setTableParams(tblProperties).build(); + client.createTable(tbl); + return client.getTable(dbName, tblName); + } + + @Test + public void testExternalSchemaAvroTable() throws TException, IOException { + //map + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_PRIMITIVE_VALUE_TYPE); + List retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "map", + retFields.get(0).getType()); + Assert.assertEquals("Unexpected comment of the field", "", retFields.get(0).getComment()); + + //list + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, + ARRAY_WITH_PRIMITIVE_ELEMENT_TYPE); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "anArray", retFields.get(0).getName()); + Assert + .assertEquals("Unexpected type of the field", "array", retFields.get(0).getType()); + + //struct + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, RECORD_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aRecord", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", + "struct", retFields.get(0).getType()); + + //union + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union-2 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_2); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union_3 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_3); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //union_4 + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, UNION_SCHEMA_4); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aUnion", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "uniontype", + retFields.get(0).getType()); + + //enum + // Enums are one of two Avro types that Hive doesn't have any native support for. + // Column names - we lose the enumness of this schema + // Column types become string + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, ENUM_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "baddies", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + // Hive has no concept of Avro's fixed type. Fixed -> arrays of bytes + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, FIXED_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "hash", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "binary", + retFields.get(0).getType()); + + //nullable string + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_STRING_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "nullableString", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + //map with nullable value - That Union[T, NULL] is converted to just T, within a Map + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, MAP_WITH_NULLABLE_PRIMITIVE_VALUE_TYPE_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "aMap", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "map", + retFields.get(0).getType()); + + // That Union[T, NULL] is converted to just T. + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, NULLABLE_ENUM_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "nullableEnum", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "string", + retFields.get(0).getType()); + + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, BYTES_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 1, retFields.size()); + Assert.assertEquals("Unexpected name of the field", "bytesField", retFields.get(0).getName()); + Assert.assertEquals("Unexpected type of the field", "binary", + retFields.get(0).getType()); + + createAvroTableWithExternalSchema(TEST_DB_NAME, TEST_TABLE_NAME, KITCHEN_SINK_SCHEMA); + retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + Assert.assertEquals("Unexpected number of fields", 16, retFields.size()); + //There are 16 fields in this schema. Instead of verifying all we verify the interesting ones + //(ones which have not been tested above) + Assert + .assertEquals("Unexpected name of 8th field", "inner_record1", retFields.get(7).getName()); + Assert.assertEquals("Unexpected type of the field", + "struct", + retFields.get(7).getType()); + + Assert.assertEquals("Unexpected field name of the 10th field", "array1", + retFields.get(9).getName()); + Assert.assertEquals("Unexpected field type of the 10th field", "array", + retFields.get(9).getType()); + + Assert.assertEquals("Unexpected field name of the 11th field", "map1", + retFields.get(10).getName()); + Assert.assertEquals("Unexpected field type of the 11th field", "map", + retFields.get(10).getType()); + + Assert.assertEquals("Unexpected field name of the 12th field", "union1", + retFields.get(11).getName()); + Assert + .assertEquals("Unexpected field type of the 12th field", "uniontype", + retFields.get(11).getType()); + + Assert.assertEquals("Unexpected field name of the 14th field", "null1", + retFields.get(13).getName()); + Assert.assertEquals("Unexpected field type of the 14th field", "void", + retFields.get(13).getType()); + + Assert.assertEquals("Unexpected field name of the 15th field", "UnionNullInt", + retFields.get(14).getName()); + Assert.assertEquals("Unexpected field type of the 15th field", "int", + retFields.get(14).getType()); + } + + private void createAvroTableWithExternalSchema(String dbName, String tblName, String schemaStr) + throws TException, IOException { + List fields = new ArrayList<>(0); + Map tblParams = new HashMap<>(); + tblParams.put(AvroSchemaUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), schemaStr); + createTable(dbName, tblName, AVRO_SERIALIZATION_LIB, fields, tblParams); + } + + @Test + public void testSimpleTable() throws TException, IOException { + List fields = new ArrayList<>(2); + FieldSchema field = new FieldSchema(); + field.setName("name"); + field.setType("string"); + field.setComment("Test name comment"); + fields.add(field); + + field = new FieldSchema(); + field.setName("age"); + field.setType("int"); + field.setComment("Test age comment"); + fields.add(field); + + createTable(TEST_DB_NAME, TEST_TABLE_NAME, null, fields, null); + List retFields = client.getFields(TEST_DB_NAME, TEST_TABLE_NAME); + verifyTableFields(fields, retFields, null); + } + + private void verifyTableFields(List expected, List actual, + String nullCommentText) { + Assert.assertEquals(expected.size(), actual.size()); + int size = expected.size(); + for (int i = 0; i < size; i++) { + FieldSchema expectedField = expected.get(i); + FieldSchema actualField = actual.get(i); + Assert.assertEquals("Name does not match for field " + (i + 1), expectedField.getName(), + actualField.getName()); + Assert.assertEquals("Type does not match for field " + (i + 1), expectedField.getType(), + actualField.getType()); + String expectedComment = null; + if (expectedField.getComment() == null && nullCommentText != null) { + expectedComment = nullCommentText; + } else { + expectedComment = expectedField.getComment(); + } + Assert.assertEquals("Comment does not match for field " + (i + 1), expectedComment, + actualField.getComment()); + } + } +} diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java new file mode 100644 index 0000000000000000000000000000000000000000..11828935f7013a0787be9490d44896fe164daffa --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/serde2/avro/TestInstanceCache.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.avro; + +import java.util.Set; + +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertSame; + +@Category(MetastoreUnitTest.class) +public class TestInstanceCache { + private static class Foo { + + private int value = 42; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Foo foo = (Foo) o; + + return value == foo.value; + + } + + @Override + public int hashCode() { + return value; + } + } + + private static class Wrapper { + public final T wrapped; + + private Wrapper(T wrapped) { + this.wrapped = wrapped; + } + } + + @Test + public void instanceCachesOnlyCreateOneInstance() throws AvroSerdeException { + InstanceCache> ic = new InstanceCache>() { + @Override + protected Wrapper makeInstance(Foo hv, + Set seenSchemas) { + return new Wrapper(hv); + } + }; + Foo f1 = new Foo(); + + Wrapper fc = ic.retrieve(f1, null); + assertSame(f1, fc.wrapped); // Our original foo should be in the wrapper + + Foo f2 = new Foo(); // Different instance, same value + + Wrapper fc2 = ic.retrieve(f2, null); + assertSame(fc2,fc); // Since equiv f, should get back first container + assertSame(fc2.wrapped, f1); + } + + @Test + public void instanceCacheReturnsCorrectInstances() throws AvroSerdeException { + InstanceCache> ic = new InstanceCache>() { + @Override + protected Wrapper makeInstance( + String hv, Set seenSchemas) { + return new Wrapper(hv); + } + }; + + Wrapper one = ic.retrieve("one", null); + Wrapper two = ic.retrieve("two", null); + + Wrapper anotherOne = ic.retrieve("one", null); + assertSame(one, anotherOne); + + Wrapper anotherTwo = ic.retrieve("two", null); + assertSame(two, anotherTwo); + } +}