diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java index 890cf4c..e5c3aa4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriter.java @@ -33,4 +33,6 @@ Object writeValue(long value) throws HiveException; Object writeValue(double value) throws HiveException; Object writeValue(byte[] value, int start, int length) throws HiveException; + Object setValue(Object row, ColumnVector column, int columnRow) throws HiveException; + Object initValue(Object ost) throws HiveException; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java index c5b026a..a242fef 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java @@ -20,8 +20,12 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; @@ -35,16 +39,39 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo; import org.apache.hadoop.io.BooleanWritable; import org.apache.hadoop.io.FloatWritable; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; /** * VectorExpressionWritableFactory helper class for generating VectorExpressionWritable objects. @@ -67,20 +94,11 @@ public ObjectInspector getObjectInspector() { return objectInspector; } - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - objectInspector = nodeDesc.getWritableObjectInspector(); - if (null == objectInspector) { - objectInspector = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo()); - } - if (null == objectInspector) { - throw new HiveException(String.format( - "Failed to initialize VectorExpressionWriter for expr: %s", - nodeDesc.getExprString())); - } + public VectorExpressionWriter init(ObjectInspector objectInspector) throws HiveException { + this.objectInspector = objectInspector; return this; } - + /** * The base implementation must be overridden by the Long specialization */ @@ -88,6 +106,13 @@ public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { public Object writeValue(long value) throws HiveException { throw new HiveException("Internal error: should not reach here"); } + + /** + * The base implementation must be overridden by the Long specialization + */ + public Object setValue(Object field, long value) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } /** * The base implementation must be overridden by the Double specialization @@ -98,12 +123,26 @@ public Object writeValue(double value) throws HiveException { } /** + * The base implementation must be overridden by the Double specialization + */ + public Object setValue(Object field, double value) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } + + /** * The base implementation must be overridden by the Bytes specialization */ @Override public Object writeValue(byte[] value, int start, int length) throws HiveException { throw new HiveException("Internal error: should not reach here"); } + + /** + * The base implementation must be overridden by the Bytes specialization + */ + public Object setValue(Object field, byte[] value, int start, int length) throws HiveException { + throw new HiveException("Internal error: should not reach here"); + } } /** @@ -133,7 +172,29 @@ public Object writeValue(ColumnVector column, int row) throws HiveException { "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0])); } - } + + @Override + public Object setValue(Object field, ColumnVector column, int row) throws HiveException { + LongColumnVector lcv = (LongColumnVector) column; + if (lcv.noNulls && !lcv.isRepeating) { + return setValue(field, lcv.vector[row]); + } else if (lcv.noNulls && lcv.isRepeating) { + return setValue(field, lcv.vector[0]); + } else if (!lcv.noNulls && !lcv.isRepeating && !lcv.isNull[row]) { + return setValue(field, lcv.vector[row]); + } else if (!lcv.noNulls && !lcv.isRepeating && lcv.isNull[row]) { + return null; + } else if (!lcv.noNulls && lcv.isRepeating && !lcv.isNull[0]) { + return setValue(field, lcv.vector[0]); + } else if (!lcv.noNulls && lcv.isRepeating && lcv.isNull[0]) { + return null; + } + throw new HiveException( + String.format( + "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", + row, lcv.noNulls, lcv.isRepeating, lcv.isNull[row], lcv.isNull[0])); + } + } /** * Specialized writer for DoubleColumnVector. Will throw cast exception @@ -161,6 +222,28 @@ public Object writeValue(ColumnVector column, int row) throws HiveException { "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0])); } + + @Override + public Object setValue(Object field, ColumnVector column, int row) throws HiveException { + DoubleColumnVector dcv = (DoubleColumnVector) column; + if (dcv.noNulls && !dcv.isRepeating) { + return setValue(field, dcv.vector[row]); + } else if (dcv.noNulls && dcv.isRepeating) { + return setValue(field, dcv.vector[0]); + } else if (!dcv.noNulls && !dcv.isRepeating && !dcv.isNull[row]) { + return setValue(field, dcv.vector[row]); + } else if (!dcv.noNulls && !dcv.isRepeating && dcv.isNull[row]) { + return null; + } else if (!dcv.noNulls && dcv.isRepeating && !dcv.isNull[0]) { + return setValue(field, dcv.vector[0]); + } else if (!dcv.noNulls && dcv.isRepeating && dcv.isNull[0]) { + return null; + } + throw new HiveException( + String.format( + "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", + row, dcv.noNulls, dcv.isRepeating, dcv.isNull[row], dcv.isNull[0])); + } } /** @@ -189,176 +272,528 @@ public Object writeValue(ColumnVector column, int row) throws HiveException { "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0])); } + + @Override + public Object setValue(Object field, ColumnVector column, int row) throws HiveException { + BytesColumnVector bcv = (BytesColumnVector) column; + if (bcv.noNulls && !bcv.isRepeating) { + return setValue(field, bcv.vector[row], bcv.start[row], bcv.length[row]); + } else if (bcv.noNulls && bcv.isRepeating) { + return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]); + } else if (!bcv.noNulls && !bcv.isRepeating && !bcv.isNull[row]) { + return setValue(field, bcv.vector[row], bcv.start[row], bcv.length[row]); + } else if (!bcv.noNulls && !bcv.isRepeating && bcv.isNull[row]) { + return null; + } else if (!bcv.noNulls && bcv.isRepeating && !bcv.isNull[0]) { + return setValue(field, bcv.vector[0], bcv.start[0], bcv.length[0]); + } else if (!bcv.noNulls && bcv.isRepeating && bcv.isNull[0]) { + return null; + } + throw new HiveException( + String.format( + "Incorrect null/repeating: row:%d noNulls:%b isRepeating:%b isNull[row]:%b isNull[0]:%b", + row, bcv.noNulls, bcv.isRepeating, bcv.isNull[row], bcv.isNull[0])); + } } - /** - * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc) - */ - public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc) - throws HiveException { - String nodeType = nodeDesc.getTypeString(); - if (nodeType.equalsIgnoreCase("tinyint")) { - return new VectorExpressionWriterLong() - { - private ByteWritable writable; - - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new ByteWritable(); - return this; - } + /** + * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc) + */ + public static VectorExpressionWriter genVectorExpressionWritable(ExprNodeDesc nodeDesc) + throws HiveException { + String nodeType = nodeDesc.getTypeString(); + ObjectInspector objectInspector = nodeDesc.getWritableObjectInspector(); + if (null == objectInspector) { + objectInspector = TypeInfoUtils + .getStandardWritableObjectInspectorFromTypeInfo(nodeDesc.getTypeInfo()); + } + if (null == objectInspector) { + throw new HiveException(String.format( + "Failed to initialize VectorExpressionWriter for expr: %s", + nodeDesc.getExprString())); + } + return genVectorExpressionWritable(objectInspector); + } - @Override - public Object writeValue(long value) { - writable.set((byte) value); - return writable; - } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("smallint")) { - return new VectorExpressionWriterLong() - { - private ShortWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new ShortWritable(); - return this; + /** + * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc) + */ + public static VectorExpressionWriter genVectorExpressionWritable( + ObjectInspector fieldObjInspector) throws HiveException { + + switch (fieldObjInspector.getCategory()) { + case PRIMITIVE: + switch (((PrimitiveObjectInspector) fieldObjInspector).getPrimitiveCategory()) { + case FLOAT: + return genVectorExpressionWritableFloat( + (SettableFloatObjectInspector) fieldObjInspector); + case DOUBLE: + return genVectorExpressionWritableDouble( + (SettableDoubleObjectInspector) fieldObjInspector); + case BOOLEAN: + return genVectorExpressionWritableBoolean( + (SettableBooleanObjectInspector) fieldObjInspector); + case BYTE: + return genVectorExpressionWritableByte( + (SettableByteObjectInspector) fieldObjInspector); + case SHORT: + return genVectorExpressionWritableShort( + (SettableShortObjectInspector) fieldObjInspector); + case INT: + return genVectorExpressionWritableInt( + (SettableIntObjectInspector) fieldObjInspector); + case LONG: + return genVectorExpressionWritableLong( + (SettableLongObjectInspector) fieldObjInspector); + case BINARY: + return genVectorExpressionWritableBinary( + (SettableBinaryObjectInspector) fieldObjInspector); + case STRING: + return genVectorExpressionWritableString( + (SettableStringObjectInspector) fieldObjInspector); + case VARCHAR: + return genVectorExpressionWritableVarchar( + (SettableHiveVarcharObjectInspector) fieldObjInspector); + case TIMESTAMP: + return genVectorExpressionWritableTimestamp( + (SettableTimestampObjectInspector) fieldObjInspector); + case DATE: + return genVectorExpressionWritableDate( + (SettableDateObjectInspector) fieldObjInspector); + case DECIMAL: + return genVectorExpressionWritableDecimal( + (SettableHiveDecimalObjectInspector) fieldObjInspector); + default: + throw new IllegalArgumentException("Unknown primitive type: " + + ((PrimitiveObjectInspector) fieldObjInspector).getPrimitiveCategory()); } + + case STRUCT: + case UNION: + case MAP: + case LIST: + throw new IllegalArgumentException("Unsupported complex type: " + + fieldObjInspector.getCategory()); + default: + throw new IllegalArgumentException("Unknown type " + + fieldObjInspector.getCategory()); + } + } - @Override - public Object writeValue(long value) { - writable.set((short) value); - return writable; - } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("int")) { - return new VectorExpressionWriterLong() - { - private IntWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new IntWritable(); - return this; - } + private static VectorExpressionWriter genVectorExpressionWritableDecimal( + SettableHiveDecimalObjectInspector fieldObjInspector) throws HiveException { + + // We should never reach this, the compile validation should guard us + throw new HiveException("DECIMAL primitive type not supported in vectorization."); + } - @Override - public Object writeValue(long value) { - writable.set((int) value); - return writable; - } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("bigint")) { - return new VectorExpressionWriterLong() - { - private LongWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new LongWritable(); - return this; - } + private static VectorExpressionWriter genVectorExpressionWritableDate( + SettableDateObjectInspector fieldObjInspector) throws HiveException { + // We should never reach this, the compile validation should guard us + throw new HiveException("DATE primitive type not supported in vectorization."); + } - @Override - public Object writeValue(long value) { - writable.set(value); - return writable; - } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("boolean")) { - return new VectorExpressionWriterLong() - { - private BooleanWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new BooleanWritable(); - return this; + private static VectorExpressionWriter genVectorExpressionWritableTimestamp( + SettableTimestampObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + private Timestamp ts; + + public VectorExpressionWriter init(SettableTimestampObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + ts = new Timestamp(0); + obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) { + TimestampUtils.assignTimeInNanoSec(value, ts); + ((SettableTimestampObjectInspector) this.objectInspector).set(obj, ts); + return obj; + } + + @Override + public Object setValue(Object field, long value) { + if (null == field) { + field = initValue(null); } + TimestampUtils.assignTimeInNanoSec(value, ts); + ((SettableTimestampObjectInspector) this.objectInspector).set(field, ts); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableTimestampObjectInspector) this.objectInspector).create(new Timestamp(0)); + } + }.init(fieldObjInspector); + } - @Override - public Object writeValue(long value) { - writable.set(value != 0 ? true : false); - return writable; + private static VectorExpressionWriter genVectorExpressionWritableVarchar( + SettableHiveVarcharObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterBytes() { + private Object obj; + private Text text; + + public VectorExpressionWriter init(SettableHiveVarcharObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.text = new Text(); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(byte[] value, int start, int length) throws HiveException { + text.set(value, start, length); + ((SettableHiveVarcharObjectInspector) this.objectInspector).set(this.obj, text.toString()); + return this.obj; + } + + @Override + public Object setValue(Object field, byte[] value, int start, int length) + throws HiveException { + if (null == field) { + field = initValue(null); } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("timestamp")) { - return new VectorExpressionWriterLong() - { - private TimestampWritable writable; - private Timestamp timestamp; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new TimestampWritable(); - timestamp = new Timestamp(0); - return this; + text.set(value, start, length); + ((SettableHiveVarcharObjectInspector) this.objectInspector).set(field, text.toString()); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableHiveVarcharObjectInspector) this.objectInspector) + .create(new HiveVarchar(StringUtils.EMPTY, -1)); + } + }.init(fieldObjInspector); + } + + private static VectorExpressionWriter genVectorExpressionWritableString( + SettableStringObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterBytes() { + private Object obj; + private Text text; + + public VectorExpressionWriter init(SettableStringObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.text = new Text(); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(byte[] value, int start, int length) throws HiveException { + this.text.set(value, start, length); + ((SettableStringObjectInspector) this.objectInspector).set(this.obj, this.text.toString()); + return this.obj; + } + + @Override + public Object setValue(Object field, byte[] value, int start, int length) + throws HiveException { + if (null == field) { + field = initValue(null); } + this.text.set(value, start, length); + ((SettableStringObjectInspector) this.objectInspector).set(field, this.text.toString()); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableStringObjectInspector) this.objectInspector).create(StringUtils.EMPTY); + } + }.init(fieldObjInspector); + } - @Override - public Object writeValue(long value) { - TimestampUtils.assignTimeInNanoSec(value, timestamp); - writable.set(timestamp); - return writable; + private static VectorExpressionWriter genVectorExpressionWritableBinary( + SettableBinaryObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterBytes() { + private Object obj; + private byte[] bytes; + + public VectorExpressionWriter init(SettableBinaryObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.bytes = ArrayUtils.EMPTY_BYTE_ARRAY; + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(byte[] value, int start, int length) throws HiveException { + bytes = Arrays.copyOfRange(value, start, start + length); + ((SettableBinaryObjectInspector) this.objectInspector).set(this.obj, bytes); + return this.obj; + } + + @Override + public Object setValue(Object field, byte[] value, int start, int length) throws HiveException { + if (null == field) { + field = initValue(null); } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("string")) { - return new VectorExpressionWriterBytes() - { - private Text writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new Text(); - return this; + bytes = Arrays.copyOfRange(value, start, start + length); + ((SettableBinaryObjectInspector) this.objectInspector).set(field, bytes); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableBinaryObjectInspector) this.objectInspector) + .create(ArrayUtils.EMPTY_BYTE_ARRAY); + } + }.init(fieldObjInspector); + } + + private static VectorExpressionWriter genVectorExpressionWritableLong( + SettableLongObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + + public VectorExpressionWriter init(SettableLongObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) throws HiveException { + ((SettableLongObjectInspector) this.objectInspector).set(this.obj, value); + return this.obj; + } + + @Override + public Object setValue(Object field, long value) throws HiveException { + if (null == field) { + field = initValue(null); } + ((SettableLongObjectInspector) this.objectInspector).set(field, value); + return field; + } - @Override - public Object writeValue(byte[] value, int start, int length) throws HiveException { - writable.set(value, start, length); - return writable; + @Override + public Object initValue(Object ignored) { + return ((SettableLongObjectInspector) this.objectInspector) + .create(0L); + } + }.init(fieldObjInspector); + } + + private static VectorExpressionWriter genVectorExpressionWritableInt( + SettableIntObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + + public VectorExpressionWriter init(SettableIntObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) throws HiveException { + ((SettableIntObjectInspector) this.objectInspector).set(this.obj, (int) value); + return this.obj; + } + + @Override + public Object setValue(Object field, long value) throws HiveException { + if (null == field) { + field = initValue(null); } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("float")) { - return new VectorExpressionWriterDouble() - { - private FloatWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new FloatWritable(); - return this; + ((SettableIntObjectInspector) this.objectInspector).set(field, (int) value); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableIntObjectInspector) this.objectInspector) + .create(0); + } + }.init(fieldObjInspector); + } + + private static VectorExpressionWriter genVectorExpressionWritableShort( + SettableShortObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + + public VectorExpressionWriter init(SettableShortObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) throws HiveException { + ((SettableShortObjectInspector) this.objectInspector).set(this.obj, (short) value); + return this.obj; + } + + @Override + public Object setValue(Object field, long value) throws HiveException { + if (null == field) { + field = initValue(null); } + ((SettableShortObjectInspector) this.objectInspector).set(field, (short) value); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableShortObjectInspector) this.objectInspector) + .create((short) 0); + } + }.init(fieldObjInspector); + } - @Override - public Object writeValue(double value) { - writable.set((float)value); - return writable; + private static VectorExpressionWriter genVectorExpressionWritableByte( + SettableByteObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + + public VectorExpressionWriter init(SettableByteObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) throws HiveException { + ((SettableByteObjectInspector) this.objectInspector).set(this.obj, (byte) value); + return this.obj; + } + + @Override + public Object setValue(Object field, long value) throws HiveException { + if (null == field) { + field = initValue(null); } - }.init(nodeDesc); - } else if (nodeType.equalsIgnoreCase("double")) { - return new VectorExpressionWriterDouble() - { - private DoubleWritable writable; - @Override - public VectorExpressionWriter init(ExprNodeDesc nodeDesc) throws HiveException { - super.init(nodeDesc); - writable = new DoubleWritable(); - return this; + ((SettableByteObjectInspector) this.objectInspector).set(field, (byte) value); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableByteObjectInspector) this.objectInspector) + .create((byte) 0); + } + }.init(fieldObjInspector); + } + + private static VectorExpressionWriter genVectorExpressionWritableBoolean( + SettableBooleanObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterLong() { + private Object obj; + + public VectorExpressionWriter init(SettableBooleanObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(long value) throws HiveException { + ((SettableBooleanObjectInspector) this.objectInspector).set(this.obj, + value == 0 ? false : true); + return this.obj; + } + + @Override + public Object setValue(Object field, long value) throws HiveException { + if (null == field) { + field = initValue(null); } + ((SettableBooleanObjectInspector) this.objectInspector).set(field, + value == 0 ? false : true); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableBooleanObjectInspector) this.objectInspector) + .create(false); + } + }.init(fieldObjInspector); + } - @Override - public Object writeValue(double value) { - writable.set(value); - return writable; + private static VectorExpressionWriter genVectorExpressionWritableDouble( + SettableDoubleObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterDouble() { + private Object obj; + + public VectorExpressionWriter init(SettableDoubleObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(double value) throws HiveException { + ((SettableDoubleObjectInspector) this.objectInspector).set(this.obj, value); + return this.obj; + } + + @Override + public Object setValue(Object field, double value) throws HiveException { + if (null == field) { + field = initValue(null); } - }.init(nodeDesc); - } + ((SettableDoubleObjectInspector) this.objectInspector).set(field, value); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableDoubleObjectInspector) this.objectInspector) + .create(0f); + } + }.init(fieldObjInspector); + } - throw new HiveException(String.format( - "Unimplemented genVectorExpressionWritable type: %s for expression: %s", - nodeType, nodeDesc)); + private static VectorExpressionWriter genVectorExpressionWritableFloat( + SettableFloatObjectInspector fieldObjInspector) throws HiveException { + return new VectorExpressionWriterDouble() { + private Object obj; + + public VectorExpressionWriter init(SettableFloatObjectInspector objInspector) + throws HiveException { + super.init(objInspector); + this.obj = initValue(null); + return this; + } + + @Override + public Object writeValue(double value) throws HiveException { + ((SettableFloatObjectInspector) this.objectInspector).set(this.obj, (float) value); + return this.obj; + } + + @Override + public Object setValue(Object field, double value) throws HiveException { + if (null == field) { + field = initValue(null); + } + ((SettableFloatObjectInspector) this.objectInspector).set(field, (float) value); + return field; + } + + @Override + public Object initValue(Object ignored) { + return ((SettableFloatObjectInspector) this.objectInspector) + .create(0f); + } + }.init(fieldObjInspector); } /** @@ -432,25 +867,87 @@ public static void processVectorExpressions( */ public static VectorExpressionWriter[] getExpressionWriters(StructObjectInspector objInspector) throws HiveException { - List outputNodeDescs = new ArrayList(); - for (StructField fieldRef : (objInspector) - .getAllStructFieldRefs()) { - String typeName = fieldRef.getFieldObjectInspector().getTypeName(); - TypeInfo ti = TypeInfoFactory.getPrimitiveTypeInfo(typeName); - outputNodeDescs.add(new ExprNodeDesc(ti) { - private static final long serialVersionUID = 1L; - - @Override - public ExprNodeDesc clone() { /* Not needed */ - return null; - } + + if (objInspector.isSettable()) { + return getSettableExpressionWriters((SettableStructObjectInspector) objInspector); + } + + List allFieldRefs = objInspector.getAllStructFieldRefs(); + + VectorExpressionWriter[] expressionWriters = new VectorExpressionWriter[allFieldRefs.size()]; + + for(int i=0; i fieldsRef = objInspector.getAllStructFieldRefs(); + VectorExpressionWriter[] writers = new VectorExpressionWriter[fieldsRef.size()]; + for(int i=0; i fieldNames1 = new ArrayList(); + fieldNames1.add("theInt"); + fieldNames1.add("theBool"); + ArrayList fieldObjectInspectors1 = new ArrayList(); + fieldObjectInspectors1 + .add(PrimitiveObjectInspectorFactory.writableIntObjectInspector); + fieldObjectInspectors1 + .add(PrimitiveObjectInspectorFactory.writableBooleanObjectInspector); + return ObjectInspectorFactory + .getStandardStructObjectInspector(fieldNames1, fieldObjectInspectors1); + } + + private void testStructLong(TypeInfo type) throws HiveException { + LongColumnVector icv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false, + vectorSize, new Random(10)); + icv.isNull[3] = true; + + LongColumnVector bcv = VectorizedRowGroupGenUtil.generateLongColumnVector(true, false, + vectorSize, new Random(10)); + bcv.isNull[2] = true; + + ArrayList[] values = (ArrayList[]) new ArrayList[this.vectorSize]; + + StructObjectInspector soi = genStructOI(); + + VectorExpressionWriter[] vew = VectorExpressionWriterFactory.getExpressionWriters(soi); + + for (int i = 0; i < vectorSize; i++) { + values[i] = new ArrayList(2); + values[i].add(null); + values[i].add(null); + + vew[0].setValue(values[i], icv, i); + vew[1].setValue(values[i], bcv, i); + + Object theInt = values[i].get(0); + if (theInt == null) { + Assert.assertTrue(icv.isNull[i]); + } else { + IntWritable w = (IntWritable) theInt; + Assert.assertEquals((int) icv.vector[i], w.get()); + } + + Object theBool = values[i].get(1); + if (theBool == null) { + Assert.assertTrue(bcv.isNull[i]); + } else { + BooleanWritable w = (BooleanWritable) theBool; + Assert.assertEquals(bcv.vector[i] == 0 ? false : true, w.get()); + } + } + } - private void testWriterBytes(TypeInfo type) throws HiveException { + private void testWriterText(TypeInfo type) throws HiveException { Text t1 = new Text("alpha"); Text t2 = new Text("beta"); BytesColumnVector bcv = new BytesColumnVector(vectorSize); @@ -160,36 +274,100 @@ private void testWriterBytes(TypeInfo type) throws HiveException { } } } - + + private void testSetterText(TypeInfo type) throws HiveException { + Text t1 = new Text("alpha"); + Text t2 = new Text("beta"); + BytesColumnVector bcv = new BytesColumnVector(vectorSize); + bcv.noNulls = false; + bcv.initBuffer(); + bcv.setVal(0, t1.getBytes(), 0, t1.getLength()); + bcv.isNull[1] = true; + bcv.setVal(2, t2.getBytes(), 0, t2.getLength()); + bcv.isNull[3] = true; + bcv.setVal(4, t1.getBytes(), 0, t1.getLength()); + + Object[] values = new Object[this.vectorSize]; + VectorExpressionWriter vew = getWriter(type); + for (int i = 0; i < vectorSize; i++) { + values[i] = vew.initValue(null); + Writable w = (Writable) vew.setValue(values[i], bcv, i); + if (w != null) { + byte [] val = new byte[bcv.length[i]]; + System.arraycopy(bcv.vector[i], bcv.start[i], val, 0, bcv.length[i]); + Writable expected = getWritableValue(type, val); + Assert.assertEquals(expected, w); + } else { + Assert.assertTrue(bcv.isNull[i]); + } + } + } + @Test public void testVectorExpressionWriterDouble() throws HiveException { testWriterDouble(TypeInfoFactory.doubleTypeInfo); } @Test + public void testVectorExpressionSetterDouble() throws HiveException { + testSetterDouble(TypeInfoFactory.doubleTypeInfo); + } + + @Test public void testVectorExpressionWriterFloat() throws HiveException { testWriterDouble(TypeInfoFactory.floatTypeInfo); } @Test + public void testVectorExpressionSetterFloat() throws HiveException { + testSetterDouble(TypeInfoFactory.floatTypeInfo); + } + + @Test public void testVectorExpressionWriterLong() throws HiveException { testWriterLong(TypeInfoFactory.longTypeInfo); } - + + @Test + public void testVectorExpressionSetterLong() throws HiveException { + testSetterLong(TypeInfoFactory.longTypeInfo); + } + + @Test + public void testVectorExpressionStructLong() throws HiveException { + testStructLong(TypeInfoFactory.longTypeInfo); + } + @Test public void testVectorExpressionWriterInt() throws HiveException { testWriterLong(TypeInfoFactory.intTypeInfo); } @Test + public void testVectorExpressionSetterInt() throws HiveException { + testSetterLong(TypeInfoFactory.intTypeInfo); + } + + @Test public void testVectorExpressionWriterShort() throws HiveException { testWriterLong(TypeInfoFactory.shortTypeInfo); } @Test + public void testVectorExpressionSetterShort() throws HiveException { + testSetterLong(TypeInfoFactory.shortTypeInfo); + } + + + @Test public void testVectorExpressionWriterBoolean() throws HiveException { testWriterLong(TypeInfoFactory.booleanTypeInfo); } + + @Test + public void testVectorExpressionSetterBoolean() throws HiveException { + testSetterLong(TypeInfoFactory.booleanTypeInfo); + } @Test public void testVectorExpressionWriterTimestamp() throws HiveException { @@ -197,12 +375,47 @@ public void testVectorExpressionWriterTimestamp() throws HiveException { } @Test - public void testVectorExpressionWriterBye() throws HiveException { + public void testVectorExpressionSetterTimestamp() throws HiveException { + testSetterLong(TypeInfoFactory.timestampTypeInfo); + } + + @Test + public void testVectorExpressionWriterByte() throws HiveException { testWriterLong(TypeInfoFactory.byteTypeInfo); } + + @Test + public void testVectorExpressionSetterByte() throws HiveException { + testSetterLong(TypeInfoFactory.byteTypeInfo); + } + + @Test + public void testVectorExpressionWriterString() throws HiveException { + testWriterText(TypeInfoFactory.stringTypeInfo); + } + + @Test + public void testVectorExpressionSetterString() throws HiveException { + testSetterText(TypeInfoFactory.stringTypeInfo); + } + + @Test + public void testVectorExpressionWriterVarchar() throws HiveException { + testWriterText(TypeInfoFactory.varcharTypeInfo); + } + + @Test + public void testVectorExpressionSetterVarchar() throws HiveException { + testSetterText(TypeInfoFactory.varcharTypeInfo); + } @Test - public void testVectorExpressionWriterBytes() throws HiveException { - testWriterBytes(TypeInfoFactory.stringTypeInfo); + public void testVectorExpressionWriterBinary() throws HiveException { + testWriterText(TypeInfoFactory.binaryTypeInfo); + } + + @Test + public void testVectorExpressionSetterBinary() throws HiveException { + testSetterText(TypeInfoFactory.binaryTypeInfo); } }