Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFComputeStats.java (working copy) @@ -138,15 +138,15 @@ } else { soi = (StructObjectInspector) parameters[0]; - countTruesField = soi.getStructFieldRef("CountTrues"); + countTruesField = soi.getStructFieldRef("counttrues"); countTruesFieldOI = (WritableLongObjectInspector) countTruesField.getFieldObjectInspector(); - countFalsesField = soi.getStructFieldRef("CountFalses"); + countFalsesField = soi.getStructFieldRef("countfalses"); countFalsesFieldOI = (WritableLongObjectInspector) countFalsesField.getFieldObjectInspector(); - countNullsField = soi.getStructFieldRef("CountNulls"); + countNullsField = soi.getStructFieldRef("countnulls"); countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); } @@ -158,10 +158,10 @@ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("CountTrues"); - fname.add("CountFalses"); - fname.add("CountNulls"); + fname.add("columntype"); + fname.add("counttrues"); + fname.add("countfalses"); + fname.add("countnulls"); partialResult = new Object[4]; partialResult[0] = new Text(); @@ -320,13 +320,13 @@ protected transient OI maxFieldOI; protected transient StructField countNullsField; - protected transient WritableLongObjectInspector countNullsFieldOI; + protected transient LongObjectInspector countNullsFieldOI; protected transient StructField ndvField; - protected transient WritableStringObjectInspector ndvFieldOI; + protected transient StringObjectInspector ndvFieldOI; protected transient StructField numBitVectorsField; - protected transient WritableIntObjectInspector numBitVectorsFieldOI; + protected transient IntObjectInspector numBitVectorsFieldOI; /* Partial aggregation result returned by TerminatePartial. Partial result is a struct * containing a long field named "count". @@ -352,20 +352,20 @@ } else { soi = (StructObjectInspector) parameters[0]; - minField = soi.getStructFieldRef("Min"); + minField = soi.getStructFieldRef("min"); minFieldOI = (OI) minField.getFieldObjectInspector(); - maxField = soi.getStructFieldRef("Max"); + maxField = soi.getStructFieldRef("max"); maxFieldOI = (OI) maxField.getFieldObjectInspector(); - countNullsField = soi.getStructFieldRef("CountNulls"); - countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); + countNullsField = soi.getStructFieldRef("countnulls"); + countNullsFieldOI = (LongObjectInspector) countNullsField.getFieldObjectInspector(); - ndvField = soi.getStructFieldRef("BitVector"); - ndvFieldOI = (WritableStringObjectInspector) ndvField.getFieldObjectInspector(); + ndvField = soi.getStructFieldRef("bitvector"); + ndvFieldOI = (StringObjectInspector) ndvField.getFieldObjectInspector(); - numBitVectorsField = soi.getStructFieldRef("NumBitVectors"); - numBitVectorsFieldOI = (WritableIntObjectInspector) + numBitVectorsField = soi.getStructFieldRef("numbitvectors"); + numBitVectorsFieldOI = (IntObjectInspector) numBitVectorsField.getFieldObjectInspector(); } @@ -380,12 +380,12 @@ foi.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("Min"); - fname.add("Max"); - fname.add("CountNulls"); - fname.add("BitVector"); - fname.add("NumBitVectors"); + fname.add("columnType"); + fname.add("min"); + fname.add("max"); + fname.add("countnulls"); + fname.add("bitvector"); + fname.add("numbitvectors"); partialResult = new Object[6]; partialResult[0] = new Text(); @@ -404,11 +404,11 @@ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("Min"); - fname.add("Max"); - fname.add("CountNulls"); - fname.add("NumDistinctValues"); + fname.add("columnType"); + fname.add("min"); + fname.add("max"); + fname.add("countnulls"); + fname.add("numdistinctvalues"); result = new Object[5]; result[0] = new Text(); @@ -706,22 +706,22 @@ private transient StructObjectInspector soi; private transient StructField maxLengthField; - private transient WritableLongObjectInspector maxLengthFieldOI; + private transient LongObjectInspector maxLengthFieldOI; private transient StructField sumLengthField; - private transient WritableLongObjectInspector sumLengthFieldOI; + private transient LongObjectInspector sumLengthFieldOI; private transient StructField countField; - private transient WritableLongObjectInspector countFieldOI; + private transient LongObjectInspector countFieldOI; private transient StructField countNullsField; - private transient WritableLongObjectInspector countNullsFieldOI; + private transient LongObjectInspector countNullsFieldOI; private transient StructField ndvField; - private transient WritableStringObjectInspector ndvFieldOI; + private transient StringObjectInspector ndvFieldOI; private transient StructField numBitVectorsField; - private transient WritableIntObjectInspector numBitVectorsFieldOI; + private transient IntObjectInspector numBitVectorsFieldOI; /* Output of final result of the aggregation */ @@ -738,23 +738,23 @@ } else { soi = (StructObjectInspector) parameters[0]; - maxLengthField = soi.getStructFieldRef("MaxLength"); - maxLengthFieldOI = (WritableLongObjectInspector) maxLengthField.getFieldObjectInspector(); + maxLengthField = soi.getStructFieldRef("maxlength"); + maxLengthFieldOI = (LongObjectInspector) maxLengthField.getFieldObjectInspector(); - sumLengthField = soi.getStructFieldRef("SumLength"); - sumLengthFieldOI = (WritableLongObjectInspector) sumLengthField.getFieldObjectInspector(); + sumLengthField = soi.getStructFieldRef("sumlength"); + sumLengthFieldOI = (LongObjectInspector) sumLengthField.getFieldObjectInspector(); - countField = soi.getStructFieldRef("Count"); - countFieldOI = (WritableLongObjectInspector) countField.getFieldObjectInspector(); + countField = soi.getStructFieldRef("count"); + countFieldOI = (LongObjectInspector) countField.getFieldObjectInspector(); - countNullsField = soi.getStructFieldRef("CountNulls"); - countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); + countNullsField = soi.getStructFieldRef("countnulls"); + countNullsFieldOI = (LongObjectInspector) countNullsField.getFieldObjectInspector(); - ndvField = soi.getStructFieldRef("BitVector"); - ndvFieldOI = (WritableStringObjectInspector) ndvField.getFieldObjectInspector(); + ndvField = soi.getStructFieldRef("bitvector"); + ndvFieldOI = (StringObjectInspector) ndvField.getFieldObjectInspector(); - numBitVectorsField = soi.getStructFieldRef("NumBitVectors"); - numBitVectorsFieldOI = (WritableIntObjectInspector) + numBitVectorsField = soi.getStructFieldRef("numbitvectors"); + numBitVectorsFieldOI = (IntObjectInspector) numBitVectorsField.getFieldObjectInspector(); } @@ -770,13 +770,13 @@ foi.add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("MaxLength"); - fname.add("SumLength"); - fname.add("Count"); - fname.add("CountNulls"); - fname.add("BitVector"); - fname.add("NumBitVectors"); + fname.add("columntype"); + fname.add("maxlength"); + fname.add("sumlength"); + fname.add("count"); + fname.add("countnulls"); + fname.add("bitvector"); + fname.add("numbitvectors"); partialResult = new Object[7]; partialResult[0] = new Text(); @@ -798,11 +798,11 @@ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("MaxLength"); - fname.add("AvgLength"); - fname.add("CountNulls"); - fname.add("NumDistinctValues"); + fname.add("columntype"); + fname.add("maxlength"); + fname.add("avglength"); + fname.add("countnulls"); + fname.add("numdistinctvalues"); result = new Object[5]; result[0] = new Text(); @@ -1030,16 +1030,16 @@ private transient StructObjectInspector soi; private transient StructField maxLengthField; - private transient WritableLongObjectInspector maxLengthFieldOI; + private transient LongObjectInspector maxLengthFieldOI; private transient StructField sumLengthField; - private transient WritableLongObjectInspector sumLengthFieldOI; + private transient LongObjectInspector sumLengthFieldOI; private transient StructField countField; - private transient WritableLongObjectInspector countFieldOI; + private transient LongObjectInspector countFieldOI; private transient StructField countNullsField; - private transient WritableLongObjectInspector countNullsFieldOI; + private transient LongObjectInspector countNullsFieldOI; /* Output of final result of the aggregation */ @@ -1055,17 +1055,17 @@ } else { soi = (StructObjectInspector) parameters[0]; - maxLengthField = soi.getStructFieldRef("MaxLength"); - maxLengthFieldOI = (WritableLongObjectInspector) maxLengthField.getFieldObjectInspector(); + maxLengthField = soi.getStructFieldRef("maxlength"); + maxLengthFieldOI = (LongObjectInspector) maxLengthField.getFieldObjectInspector(); - sumLengthField = soi.getStructFieldRef("SumLength"); - sumLengthFieldOI = (WritableLongObjectInspector) sumLengthField.getFieldObjectInspector(); + sumLengthField = soi.getStructFieldRef("sumlength"); + sumLengthFieldOI = (LongObjectInspector) sumLengthField.getFieldObjectInspector(); - countField = soi.getStructFieldRef("Count"); - countFieldOI = (WritableLongObjectInspector) countField.getFieldObjectInspector(); + countField = soi.getStructFieldRef("count"); + countFieldOI = (LongObjectInspector) countField.getFieldObjectInspector(); - countNullsField = soi.getStructFieldRef("CountNulls"); - countNullsFieldOI = (WritableLongObjectInspector) countNullsField.getFieldObjectInspector(); + countNullsField = soi.getStructFieldRef("countnulls"); + countNullsFieldOI = (LongObjectInspector) countNullsField.getFieldObjectInspector(); } @@ -1079,11 +1079,11 @@ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("MaxLength"); - fname.add("SumLength"); - fname.add("Count"); - fname.add("CountNulls"); + fname.add("columntype"); + fname.add("maxlength"); + fname.add("sumlength"); + fname.add("count"); + fname.add("countnulls"); partialResult = new Object[5]; partialResult[0] = new Text(); @@ -1102,10 +1102,10 @@ foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector); List fname = new ArrayList(); - fname.add("ColumnType"); - fname.add("MaxLength"); - fname.add("AvgLength"); - fname.add("CountNulls"); + fname.add("columntype"); + fname.add("maxlength"); + fname.add("avglength"); + fname.add("countnulls"); result = new Object[4]; result[0] = new Text(); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFContextNGrams.java (working copy) @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -158,16 +159,16 @@ */ public static class GenericUDAFContextNGramEvaluator extends GenericUDAFEvaluator { // For PARTIAL1 and COMPLETE: ObjectInspectors for original data - private transient StandardListObjectInspector outerInputOI; + private transient ListObjectInspector outerInputOI; private transient StandardListObjectInspector innerInputOI; - private transient StandardListObjectInspector contextListOI; + private transient ListObjectInspector contextListOI; private PrimitiveObjectInspector contextOI; private PrimitiveObjectInspector inputOI; private transient PrimitiveObjectInspector kOI; private transient PrimitiveObjectInspector pOI; // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations - private transient StandardListObjectInspector loi; + private transient ListObjectInspector loi; @Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { @@ -175,7 +176,7 @@ // Init input object inspectors if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { - outerInputOI = (StandardListObjectInspector) parameters[0]; + outerInputOI = (ListObjectInspector) parameters[0]; if(outerInputOI.getListElementObjectInspector().getCategory() == ObjectInspector.Category.LIST) { // We're dealing with input that is an array of arrays of strings @@ -186,7 +187,7 @@ inputOI = (PrimitiveObjectInspector) outerInputOI.getListElementObjectInspector(); innerInputOI = null; } - contextListOI = (StandardListObjectInspector) parameters[1]; + contextListOI = (ListObjectInspector) parameters[1]; contextOI = (PrimitiveObjectInspector) contextListOI.getListElementObjectInspector(); kOI = (PrimitiveObjectInspector) parameters[2]; if(parameters.length == 4) { @@ -196,7 +197,7 @@ } } else { // Init the list object inspector for handling partial aggregations - loi = (StandardListObjectInspector) parameters[0]; + loi = (ListObjectInspector) parameters[0]; } // Init output object inspectors. @@ -229,10 +230,10 @@ return; } NGramAggBuf myagg = (NGramAggBuf) agg; - List partial = (List) loi.getList(obj); + List partial = (List) loi.getList(obj); // remove the context words from the end of the list - int contextSize = Integer.parseInt( ((Text)partial.get(partial.size()-1)).toString() ); + int contextSize = Integer.parseInt( partial.get(partial.size()-1).toString() ); partial.remove(partial.size()-1); if(myagg.context.size() > 0) { if(contextSize != myagg.context.size()) { Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java (working copy) @@ -88,8 +88,9 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { super.init(m, parameters); - partialCountAggOI = - PrimitiveObjectInspectorFactory.writableLongObjectInspector; + if (mode == Mode.PARTIAL2 || mode == Mode.FINAL) { + partialCountAggOI = (LongObjectInspector)parameters[0]; + } result = new LongWritable(0); return PrimitiveObjectInspectorFactory.writableLongObjectInspector; } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java (working copy) @@ -91,12 +91,12 @@ .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); } else if (m == Mode.PARTIAL2 || m == Mode.FINAL) { internalMergeOI = (StandardListObjectInspector) parameters[0]; - inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; + inputOI = (PrimitiveObjectInspector)internalMergeOI.getListElementObjectInspector(); loi = (StandardListObjectInspector) ObjectInspectorFactory .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); return loi; } else { // Mode.COMPLETE, ie. no map-side aggregation, requires ordering - inputOI = PrimitiveObjectInspectorFactory.writableByteObjectInspector; + inputOI = (PrimitiveObjectInspector)parameters[0]; loi = (StandardListObjectInspector) ObjectInspectorFactory .getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector); return loi; Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java (working copy) @@ -28,10 +28,11 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; @@ -124,7 +125,7 @@ private transient PrimitiveObjectInspector nbinsOI; // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations (list of doubles) - private transient StandardListObjectInspector loi; + private transient ListObjectInspector loi; @Override @@ -137,7 +138,7 @@ inputOI = (PrimitiveObjectInspector) parameters[0]; nbinsOI = (PrimitiveObjectInspector) parameters[1]; } else { - loi = (StandardListObjectInspector) parameters[0]; + loi = (ListObjectInspector) parameters[0]; } // init output object inspectors @@ -197,8 +198,10 @@ return; } List partialHistogram = (List) loi.getList(partial); + DoubleObjectInspector doi = (DoubleObjectInspector)loi.getListElementObjectInspector(); + StdAgg myagg = (StdAgg) agg; - myagg.histogram.merge(partialHistogram); + myagg.histogram.merge(partialHistogram, doi); } @Override Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java (working copy) @@ -25,6 +25,7 @@ import java.util.List; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -44,7 +45,7 @@ // of objs) private transient StandardListObjectInspector loi; - private transient StandardListObjectInspector internalMergeOI; + private transient ListObjectInspector internalMergeOI; private BufferType bufferType; @@ -68,14 +69,14 @@ .getStandardListObjectInspector((PrimitiveObjectInspector) ObjectInspectorUtils .getStandardObjectInspector(inputOI)); } else { - if (!(parameters[0] instanceof StandardListObjectInspector)) { + if (!(parameters[0] instanceof ListObjectInspector)) { //no map aggregation. inputOI = (PrimitiveObjectInspector) ObjectInspectorUtils .getStandardObjectInspector(parameters[0]); return (StandardListObjectInspector) ObjectInspectorFactory .getStandardListObjectInspector(inputOI); } else { - internalMergeOI = (StandardListObjectInspector) parameters[0]; + internalMergeOI = (ListObjectInspector) parameters[0]; inputOI = (PrimitiveObjectInspector) internalMergeOI.getListElementObjectInspector(); loi = (StandardListObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI); return loi; Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java (working copy) @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -300,20 +301,21 @@ return; } PercentileAggBuf myagg = (PercentileAggBuf) agg; - List partialHistogram = (List) loi.getList(partial); + List partialHistogram = (List) loi.getList(partial); + DoubleObjectInspector doi = (DoubleObjectInspector)loi.getListElementObjectInspector(); // remove requested quantiles from the head of the list - int nquantiles = (int) partialHistogram.get(0).get(); + int nquantiles = (int) doi.get(partialHistogram.get(0)); if(nquantiles > 0) { myagg.quantiles = new double[nquantiles]; for(int i = 1; i <= nquantiles; i++) { - myagg.quantiles[i-1] = partialHistogram.get(i).get(); + myagg.quantiles[i-1] = doi.get(partialHistogram.get(i)); } partialHistogram.subList(0, nquantiles+1).clear(); } // merge histograms - myagg.histogram.merge(partialHistogram); + myagg.histogram.merge(partialHistogram, doi); } @Override Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java (working copy) @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; @@ -160,7 +161,7 @@ */ public static class GenericUDAFnGramEvaluator extends GenericUDAFEvaluator { // For PARTIAL1 and COMPLETE: ObjectInspectors for original data - private transient StandardListObjectInspector outerInputOI; + private transient ListObjectInspector outerInputOI; private transient StandardListObjectInspector innerInputOI; private transient PrimitiveObjectInspector inputOI; private transient PrimitiveObjectInspector nOI; @@ -168,7 +169,7 @@ private transient PrimitiveObjectInspector pOI; // For PARTIAL2 and FINAL: ObjectInspectors for partial aggregations - private transient StandardListObjectInspector loi; + private transient ListObjectInspector loi; @Override public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { @@ -176,7 +177,7 @@ // Init input object inspectors if (m == Mode.PARTIAL1 || m == Mode.COMPLETE) { - outerInputOI = (StandardListObjectInspector) parameters[0]; + outerInputOI = (ListObjectInspector) parameters[0]; if(outerInputOI.getListElementObjectInspector().getCategory() == ObjectInspector.Category.LIST) { // We're dealing with input that is an array of arrays of strings @@ -196,7 +197,7 @@ } } else { // Init the list object inspector for handling partial aggregations - loi = (StandardListObjectInspector) parameters[0]; + loi = (ListObjectInspector) parameters[0]; } // Init output object inspectors. @@ -229,7 +230,7 @@ return; } NGramAggBuf myagg = (NGramAggBuf) agg; - List partialNGrams = (List) loi.getList(partial); + List partialNGrams = (List) loi.getList(partial); int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString()); if(myagg.n > 0 && myagg.n != n) { throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'" Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCase.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCase.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCase.java (working copy) @@ -101,7 +101,7 @@ PrimitiveObjectInspector caseOI = (PrimitiveObjectInspector) caseOIResolver.get(); if (PrimitiveObjectInspectorUtils.comparePrimitiveObjects( caseOIResolver.convertIfNecessary(exprValue, argumentOIs[0]), caseOI, - caseOIResolver.convertIfNecessary(caseKey, argumentOIs[i]), caseOI)) { + caseOIResolver.convertIfNecessary(caseKey, argumentOIs[i], false), caseOI)) { Object caseValue = arguments[i + 1].get(); return returnOIResolver.convertIfNecessary(caseValue, argumentOIs[i + 1]); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFIn.java (working copy) @@ -181,7 +181,7 @@ conversionHelper.convertIfNecessary( arguments[0].get(), argumentOIs[0]), compareOI, conversionHelper.convertIfNecessary( - arguments[i].get(), argumentOIs[i]), compareOI) == 0) { + arguments[i].get(), argumentOIs[i], false), compareOI) == 0) { bw.set(true); return bw; } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNamedStruct.java (working copy) @@ -19,16 +19,15 @@ package org.apache.hadoop.hive.ql.udf.generic; import java.util.ArrayList; -import java.util.Arrays; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; import org.apache.hadoop.hive.ql.exec.Description; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @Description(name = "named_struct", @@ -51,12 +50,12 @@ ArrayList fname = new ArrayList(numFields / 2); ArrayList retOIs = new ArrayList(numFields / 2); for (int f = 0; f < numFields; f+=2) { - if (!(arguments[f] instanceof WritableConstantStringObjectInspector)) { + if (!(arguments[f] instanceof ConstantObjectInspector)) { throw new UDFArgumentTypeException(f, "Even arguments" + " to NAMED_STRUCT must be a constant STRING." + arguments[f].toString()); } - WritableConstantStringObjectInspector constantOI = - (WritableConstantStringObjectInspector)arguments[f]; + ConstantObjectInspector constantOI = + (ConstantObjectInspector)arguments[f]; fname.add(constantOI.getWritableConstantValue().toString()); retOIs.add(arguments[f + 1]); } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java (working copy) @@ -197,12 +197,18 @@ public ObjectInspector get(ObjectInspector defaultOI) { return returnObjectInspector != null ? returnObjectInspector : defaultOI; } + + public Object convertIfNecessary(Object o, ObjectInspector oi) { + return convertIfNecessary(o, oi, true); + } /** * Convert the return Object if necessary (when the ObjectInspectors of - * different possibilities are not all the same). + * different possibilities are not all the same). If reuse is true, + * the result Object will be the same object as the last invocation + * (as long as the oi is the same) */ - public Object convertIfNecessary(Object o, ObjectInspector oi) { + public Object convertIfNecessary(Object o, ObjectInspector oi, boolean reuse) { Object converted = null; if (oi == returnObjectInspector) { converted = o; @@ -212,11 +218,14 @@ return null; } - if (converters == null) { - converters = new HashMap(); + Converter converter = null; + if (reuse) { + if (converters == null) { + converters = new HashMap(); + } + converter = converters.get(oi); } - Converter converter = converters.get(oi); if (converter == null) { converter = ObjectInspectorConverters.getConverter(oi, returnObjectInspector); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFStack.java (working copy) @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ReturnObjectInspectorResolver; +import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -60,13 +61,13 @@ if (args.length < 2) { throw new UDFArgumentException("STACK() expects at least two arguments."); } - if (!(args[0] instanceof WritableConstantIntObjectInspector)) { + if (!(args[0] instanceof ConstantObjectInspector)) { throw new UDFArgumentException( "The first argument to STACK() must be a constant integer (got " + args[0].getTypeName() + " instead)."); } - numRows = - ((WritableConstantIntObjectInspector)args[0]).getWritableConstantValue(); + numRows = (IntWritable) + ((ConstantObjectInspector)args[0]).getWritableConstantValue(); if (numRows == null || numRows.get() < 1) { throw new UDFArgumentException( Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NGramEstimator.java (working copy) @@ -208,7 +208,7 @@ * * @param other A serialized n-gram object created by the serialize() method */ - public void merge(List other) throws HiveException { + public void merge(List other) throws HiveException { if(other == null) { return; } @@ -240,8 +240,7 @@ for(int i = 3; i < other.size(); i++) { ArrayList key = new ArrayList(); for(int j = 0; j < n; j++) { - Text word = other.get(i+j); - key.add(word.toString()); + key.add(other.get(i+j).toString()); } i += n; double val = Double.parseDouble( other.get(i).toString() ); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java (revision 1661840) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java (working copy) @@ -21,7 +21,9 @@ import java.util.List; import java.util.Collections; import java.util.Random; + import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; /** @@ -115,7 +117,7 @@ * @param other A serialized histogram created by the serialize() method * @see #merge */ - public void merge(List other) { + public void merge(List other, DoubleObjectInspector doi) { if(other == null) { return; } @@ -123,13 +125,13 @@ if(nbins == 0 || nusedbins == 0) { // Our aggregation buffer has nothing in it, so just copy over 'other' // by deserializing the ArrayList of (x,y) pairs into an array of Coord objects - nbins = (int) other.get(0).get(); + nbins = (int) doi.get(other.get(0)); nusedbins = (other.size()-1)/2; bins = new ArrayList(nusedbins); for (int i = 1; i < other.size(); i+=2) { Coord bin = new Coord(); - bin.x = other.get(i).get(); - bin.y = other.get(i+1).get(); + bin.x = doi.get(other.get(i)); + bin.y = doi.get(other.get(i+1)); bins.add(bin); } } else { @@ -146,8 +148,8 @@ } for (int j = 1; j < other.size(); j += 2) { Coord bin = new Coord(); - bin.x = other.get(j).get(); - bin.y = other.get(j+1).get(); + bin.x = doi.get(other.get(j)); + bin.y = doi.get(other.get(j+1)); tmp_bins.add(bin); } Collections.sort(tmp_bins);