diff --git metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index a65c6b6..dfd710a 100644 --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -66,6 +66,8 @@ public static final String META_TABLE_SERDE = "serde"; public static final String META_TABLE_PARTITION_COLUMNS = "partition_columns"; + + public static final String META_TABLE_PARTITION_COLUMN_TYPES = "partition_columns.types"; public static final String FILE_INPUT_FORMAT = "file.inputformat"; diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index cee1637..eec5b1c 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -953,11 +953,16 @@ public static Properties getSchema( String partString = ""; String partStringSep = ""; + String partTypesString = ""; + String partTypesStringSep = ""; for (FieldSchema partKey : partitionKeys) { partString = partString.concat(partStringSep); partString = partString.concat(partKey.getName()); + partTypesString = partTypesString.concat(partTypesStringSep); + partTypesString = partTypesString.concat(partKey.getType()); if (partStringSep.length() == 0) { partStringSep = "/"; + partTypesStringSep = ":"; } } if (partString.length() > 0) { @@ -965,6 +970,10 @@ public static Properties getSchema( .setProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS, partString); + schema + .setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES, + partTypesString); } if (parameters != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index ffc4c42..adae26d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -59,6 +59,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.InputFormat; @@ -245,9 +248,12 @@ private StructObjectInspector getRowInspectorFromPartition(PartitionDesc partiti String pcols = partition.getTableDesc().getProperties().getProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS); String[] partKeys = pcols.trim().split("/"); - row[1] = createPartValue(partKeys, partition.getPartSpec()); + String pcolTypes = partition.getTableDesc().getProperties().getProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); + String[] partKeyTypes = pcolTypes.trim().split(":"); + row[1] = createPartValue(partKeys, partition.getPartSpec(), partKeyTypes); - return createRowInspector(getStructOIFrom(partitionOI), partKeys); + return createRowInspector(getStructOIFrom(partitionOI), partKeys, partKeyTypes); } private StructObjectInspector getRowInspectorFromPartitionedTable(TableDesc table) @@ -257,8 +263,11 @@ private StructObjectInspector getRowInspectorFromPartitionedTable(TableDesc tabl String pcols = table.getProperties().getProperty( org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS); String[] partKeys = pcols.trim().split("/"); + String pcolTypes = table.getProperties().getProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); + String[] partKeyTypes = pcolTypes.trim().split(":"); row[1] = null; - return createRowInspector(getStructOIFrom(serde.getObjectInspector()), partKeys); + return createRowInspector(getStructOIFrom(serde.getObjectInspector()), partKeys, partKeyTypes); } private StructObjectInspector getStructOIFrom(ObjectInspector current) throws SerDeException { @@ -276,13 +285,22 @@ private StructObjectInspector createRowInspector(StructObjectInspector current) Arrays.asList(current, vcsOI)) : current; } - private StructObjectInspector createRowInspector(StructObjectInspector current, String[] partKeys) + private StructObjectInspector createRowInspector(StructObjectInspector current, String[] partKeys, String[] partKeyTypes) throws SerDeException { List partNames = new ArrayList(); List partObjectInspectors = new ArrayList(); - for (String key : partKeys) { + for (int i = 0; i < partKeys.length; i++) { + String key = partKeys[i]; partNames.add(key); - partObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector); + PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils. + getTypeEntryFromTypeName(PrimitiveObjectInspectorUtils. + getTypeNameFromColumnTypes(partKeyTypes[i])); + ObjectInspector oi = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + if (te != null) { + oi = PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(te.primitiveCategory); + } + partObjectInspectors.add(oi); } StructObjectInspector partObjectInspector = ObjectInspectorFactory .getStandardStructObjectInspector(partNames, partObjectInspectors); @@ -292,10 +310,32 @@ private StructObjectInspector createRowInspector(StructObjectInspector current, Arrays.asList(current, partObjectInspector)); } - private List createPartValue(String[] partKeys, Map partSpec) { - List partValues = new ArrayList(); - for (String key : partKeys) { - partValues.add(partSpec.get(key)); + private Object[] createPartValue(String[] partKeys, Map partSpec, String[] partKeyTypes) { + Object[] partValues = new Object[partKeys.length]; + for (int i = 0; i < partKeys.length; i++) { + String key = partKeys[i]; + PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils. + getTypeEntryFromTypeName(PrimitiveObjectInspectorUtils. + getTypeNameFromColumnTypes(partKeyTypes[i])); + ObjectInspector oi = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + // Partitions do not exist for this table + if (partSpec == null) { + // for partitionless table, initialize partValue to null + partValues[i] = null; + } else { + partValues[i] = new Text(partSpec.get(key)); + // If we were not able to map te to a primitive type, just store the values as a text. + if (te == null) { + partValues[i] = new Text(partSpec.get(key)); + } else { + oi = PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(te.primitiveCategory); + partValues[i] = + ObjectInspectorConverters. + getConverter(PrimitiveObjectInspectorFactory. + javaStringObjectInspector, oi).convert(partSpec.get(key)); + } + } } return partValues; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java index 5b6dd6a..1a0b6cf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java @@ -48,11 +48,20 @@ import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.IdentityConverter; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; @@ -179,7 +188,7 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, PartitionDesc pd = ctx.partDesc; TableDesc td = pd.getTableDesc(); - + MapOpCtx opCtx = new MapOpCtx(); // Use table properties in case of unpartitioned tables, // and the union of table properties and partition properties, with partition @@ -203,28 +212,52 @@ private MapOpCtx initObjectInspector(Configuration hconf, MapInputPath ctx, opCtx.partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter( partRawRowObjectInspector, opCtx.tblRawRowObjectInspector); - + // Next check if this table has partitions and if so // get the list of partition names as well as allocate // the serdes for the partition columns String pcols = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMNS); - // Log LOG = LogFactory.getLog(MapOperator.class.getName()); + if (pcols != null && pcols.length() > 0) { String[] partKeys = pcols.trim().split("/"); + String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); + String[] partKeyTypes = pcolTypes.trim().split(":"); + + if (partKeys.length != partKeyTypes.length) { + throw new HiveException("Internal error : partKeys length, " +partKeys.length + + " not the same as partKeyTypes length, " + partKeyTypes.length); + } + List partNames = new ArrayList(partKeys.length); Object[] partValues = new Object[partKeys.length]; List partObjectInspectors = new ArrayList(partKeys.length); + for (int i = 0; i < partKeys.length; i++) { String key = partKeys[i]; partNames.add(key); + PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils. + getTypeEntryFromTypeName(PrimitiveObjectInspectorUtils. + getTypeNameFromColumnTypes(partKeyTypes[i])); + ObjectInspector oi = PrimitiveObjectInspectorFactory.writableStringObjectInspector; + // Partitions do not exist for this table if (partSpec == null) { // for partitionless table, initialize partValue to null partValues[i] = null; } else { - partValues[i] = new Text(partSpec.get(key)); + // If we were not able to map te to a primitive type, just store the values as a text. + if (te == null) { + partValues[i] = new Text(partSpec.get(key)); + } else { + oi = PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(te.primitiveCategory); + partValues[i] = + ObjectInspectorConverters. + getConverter(PrimitiveObjectInspectorFactory. + javaStringObjectInspector, oi).convert(partSpec.get(key)); + } } - partObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); + partObjectInspectors.add(oi); } opCtx.rowWithPart = new Object[] {null, partValues}; opCtx.partObjectInspector = ObjectInspectorFactory diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java index a7efa41..659ea08 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector; import java.io.IOException; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashMap; @@ -27,7 +28,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.SerializationUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.type.Decimal128; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -38,14 +41,24 @@ import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.FileSplit; +import org.mortbay.log.Log; + +import sun.util.LocaleServiceProviderPool.LocalizedObjectGetter; /** * Context for Vectorized row batch. this calss does eager deserialization of row data using serde @@ -54,7 +67,7 @@ * with the partition column. */ public class VectorizedRowBatchCtx { - + // OI for raw row data (EG without partition cols) private StructObjectInspector rawRowOI; @@ -65,8 +78,14 @@ private Deserializer deserializer; // Hash map of partition values. Key=TblColName value=PartitionValue - private Map partitionValues; - + private Map partitionValues; + + // partition types + private Map partitionTypes; + + // converter types to add row to context + private Map partitionColConverters; + // Column projection list - List of column indexes to include. This // list does not contain partition columns private List colsToInclude; @@ -86,11 +105,13 @@ * Hash map of partition values. Key=TblColName value=PartitionValue */ public VectorizedRowBatchCtx(StructObjectInspector rawRowOI, StructObjectInspector rowOI, - Deserializer deserializer, Map partitionValues) { + Deserializer deserializer, Map partitionValues, + Map partitionTypes) { this.rowOI = rowOI; this.rawRowOI = rawRowOI; this.deserializer = deserializer; this.partitionValues = partitionValues; + this.partitionTypes = partitionTypes; } /** @@ -173,25 +194,54 @@ public void init(Configuration hiveConf, FileSplit split) throws ClassNotFoundEx // raw row object inspector (row with out partition col) LinkedHashMap partSpec = part.getPartSpec(); String[] partKeys = pcols.trim().split("/"); + String pcolTypes = partProps.getProperty(hive_metastoreConstants.META_TABLE_PARTITION_COLUMN_TYPES); + String[] partKeyTypes = pcolTypes.trim().split(":"); + + if (partKeys.length != partKeyTypes.length) { + throw new HiveException("Internal error : partKeys length, " +partKeys.length + + " not the same as partKeyTypes length, " + partKeyTypes.length); + } + List partNames = new ArrayList(partKeys.length); - partitionValues = new LinkedHashMap(); - List partObjectInspectors = new ArrayList( - partKeys.length); + List partObjectInspectors = new ArrayList(partKeys.length); + partitionValues = new LinkedHashMap(); + partitionTypes = new LinkedHashMap(); + partitionColConverters = new LinkedHashMap(); + Map categoryToConverter = new LinkedHashMap(); for (int i = 0; i < partKeys.length; i++) { String key = partKeys[i]; - partNames.add(key); + partNames.add(i, key); + ObjectInspector objectInspector = null; + Object objectVal; if (partSpec == null) { // for partitionless table, initialize partValue to empty string. // We can have partitionless table even if we have partition keys // when there is only only partition selected and the partition key is not // part of the projection/include list. - partitionValues.put(key, ""); + objectVal = ""; + partitionTypes.put(key, PrimitiveCategory.STRING); + objectInspector = PrimitiveObjectInspectorFactory.writableStringObjectInspector; } else { - partitionValues.put(key, partSpec.get(key)); + PrimitiveTypeEntry te = PrimitiveObjectInspectorUtils. + getTypeEntryFromTypeName(PrimitiveObjectInspectorUtils. + getTypeNameFromColumnTypes(partKeyTypes[i])); + if (te == null) { + throw new HiveException("Unsupported partition key type " + partKeyTypes[i]); + } else { + objectInspector = PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(te.primitiveCategory); + objectVal = + ObjectInspectorConverters. + getConverter(PrimitiveObjectInspectorFactory. + javaStringObjectInspector, objectInspector). + convert(partSpec.get(key)); + partitionTypes.put(key, te.primitiveCategory); + partitionColConverters.put(key, + getRunTimeOIConverter(te.primitiveCategory, categoryToConverter)); + } } - - partObjectInspectors - .add(PrimitiveObjectInspectorFactory.writableStringObjectInspector); + partitionValues.put(key, objectVal); + partObjectInspectors.add(i, objectInspector); } // Create partition OI @@ -213,7 +263,7 @@ public void init(Configuration hiveConf, FileSplit split) throws ClassNotFoundEx colsToInclude = ColumnProjectionUtils.getReadColumnIDs(hiveConf); } - + /** * Creates a Vectorized row batch and the column vectors. * @@ -274,8 +324,7 @@ public VectorizedRowBatch createVectorizedRowBatch() throws HiveException + foi.getCategory()); default: throw new HiveException("Unknown ObjectInspector category!"); - - } + } } } result.numCols = fieldRefs.size(); @@ -334,7 +383,54 @@ private int getColIndexBasedOnColName(String colName) throws HiveException } throw new HiveException("Not able to find column name in row object inspector"); } + + /** Map the primitive category to runtime converter used to interpret the object at runtime. + * @throws HiveException + * + */ + private Converter getRunTimeOIConverter(PrimitiveCategory pCategory, + Map categoryToConverter) throws HiveException { + if (categoryToConverter.containsKey(pCategory)) { + return categoryToConverter.get(pCategory); + } + Converter ret = null; + + switch (pCategory) { + case BOOLEAN: + case BYTE: + case SHORT: + case INT: + case LONG: + case DATE: + case TIMESTAMP: + ret = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(pCategory), + PrimitiveObjectInspectorFactory.javaLongObjectInspector); + break; + case FLOAT: + case DOUBLE: + ret = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(pCategory), + PrimitiveObjectInspectorFactory.javaDoubleObjectInspector); + break; + case DECIMAL: + ret = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(pCategory), + PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector); + break; + case STRING: + ret = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory. + getPrimitiveWritableObjectInspector(pCategory), + PrimitiveObjectInspectorFactory.javaStringObjectInspector); + break; + default: + throw new HiveException("Unable to recognize the partition type " + pCategory); + } + categoryToConverter.put(pCategory, ret); + return ret; + } + /** * Add the partition values to the batch * @@ -344,17 +440,75 @@ private int getColIndexBasedOnColName(String colName) throws HiveException public void addPartitionColsToBatch(VectorizedRowBatch batch) throws HiveException { int colIndex; - String value; - BytesColumnVector bcv; + Object value; + PrimitiveCategory pCategory; if (partitionValues != null) { for (String key : partitionValues.keySet()) { colIndex = getColIndexBasedOnColName(key); value = partitionValues.get(key); - bcv = (BytesColumnVector) batch.cols[colIndex]; - bcv.setRef(0, value.getBytes(), 0, value.length()); - bcv.isRepeating = true; - bcv.isNull[0] = false; - bcv.noNulls = true; + pCategory = partitionTypes.get(key); + + switch (pCategory) { + case BOOLEAN: + case BYTE: + case SHORT: + case INT: + case LONG: + case DATE: + case TIMESTAMP: + LongColumnVector lcv = (LongColumnVector) batch.cols[colIndex]; + Object lVal = partitionColConverters.get(key).convert(value); + if (lVal == null) { + lcv.noNulls = false; + lcv.isNull[0] = true; + lcv.isRepeating = true; + } else { + lcv.fill((Long)lVal); + lcv.isNull[0] = false; + } + break; + case FLOAT: + case DOUBLE: + DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[colIndex]; + Object dVal = partitionColConverters.get(key).convert(value); + if (dVal == null) { + dcv.noNulls = false; + dcv.isNull[0] = true; + dcv.isRepeating = true; + } else { + dcv.fill((Double) dVal); + dcv.isNull[0] = false; + } + break; + case DECIMAL: + DecimalColumnVector dv = (DecimalColumnVector) batch.cols[colIndex]; + Object decVal = partitionColConverters.get(key).convert(value); + if (decVal == null) { + dv.noNulls = false; + dv.isNull[0] = true; + dv.isRepeating = true; + } else { + dv.vector[0] = (Decimal128) decVal; + dv.isRepeating = true; + dv.isNull[0] = false; + } + break; + case STRING: + BytesColumnVector bcv = (BytesColumnVector) batch.cols[colIndex]; + Object sVal = partitionColConverters.get(key).convert(value); + if (sVal == null) { + bcv.noNulls = false; + bcv.isNull[0] = true; + bcv.isRepeating = true; + } else { + bcv.fill(((String) sVal).getBytes()); + bcv.isNull[0] = false; + } + break; + default: + throw new HiveException("Unable to recognize the partition type " + pCategory + + " for column " + key); + } } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index e1e427f..f2eeddc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -8590,7 +8590,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // TODO: use the right type by calling part_col.getType() instead of // String.class. See HIVE-3059. rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), - TypeInfoFactory.stringTypeInfo, alias, true)); + TypeInfoFactory.getPrimitiveTypeInfo(part_col.getType()), alias, true)); } // put all virutal columns in RowResolver. diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java index efbc758..80e64d9 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java @@ -202,7 +202,7 @@ private VectorizedRowBatch GetRowBatch() throws SerDeException, HiveException, I Assert.assertEquals("Field size should be 9", colCount, fieldRefs.size()); // Create the context - VectorizedRowBatchCtx ctx = new VectorizedRowBatchCtx(oi, oi, serDe, null); + VectorizedRowBatchCtx ctx = new VectorizedRowBatchCtx(oi, oi, serDe, null, null); VectorizedRowBatch batch = ctx.createVectorizedRowBatch(); VectorizedBatchUtil.SetNoNullFields(true, batch); diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 5ccacf1..c45c8e9 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -26,6 +26,7 @@ import java.sql.Timestamp; import java.util.HashMap; import java.util.Map; +import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,6 +58,8 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; +import sun.rmi.rmic.iiop.PrimitiveType; + /** * ObjectInspectorFactory is the primary way to create new ObjectInspector * instances. @@ -1155,7 +1158,28 @@ public static PrimitiveGrouping getPrimitiveGrouping(PrimitiveCategory primitive return PrimitiveGrouping.UNKNOWN_GROUP; } } + + public static String getTypeNameFromColumnTypes(String columnType) { + Pattern decimalTypePattern = Pattern.compile("decimal.*", + Pattern.CASE_INSENSITIVE); + Pattern charTypePattern = Pattern.compile("char.*", + Pattern.CASE_INSENSITIVE); + Pattern varcharTypePattern = Pattern.compile("varchar.*", + Pattern.CASE_INSENSITIVE); + if (decimalTypePattern.matcher(columnType).matches()) { + return serdeConstants.DECIMAL_TYPE_NAME; + } + if (charTypePattern.matcher(columnType).matches()) { + return serdeConstants.CHAR_TYPE_NAME; + } + if (varcharTypePattern.matcher(columnType).matches()) { + return serdeConstants.VARCHAR_TYPE_NAME; + } + // Return the column type as it is defined. + return columnType; + } + private PrimitiveObjectInspectorUtils() { // prevent instantiation }