From 31dbdff9be8898aab875854ba768adef2dc23a93 Mon Sep 17 00:00:00 2001 From: Gopal V Date: Fri, 15 Jan 2016 00:16:49 -0800 Subject: [PATCH] test cases populate isNull in the vector movement operations --- .../hadoop/hive/ql/exec/vector/VectorAssignRow.java | 16 ++++++++++++++++ .../hadoop/hive/ql/exec/vector/VectorCopyRow.java | 10 ++++++++++ .../hive/ql/exec/vector/VectorDeserializeRow.java | 20 ++++++++++++++++++++ .../hive/ql/exec/vector/TestVectorRowObject.java | 7 +++++++ .../hive/ql/exec/vector/TestVectorSerDeRow.java | 10 ++++++++++ 5 files changed, 63 insertions(+) diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java index 92b4a07..d69454f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java @@ -139,6 +139,7 @@ void assign(int batchIndex, Object object) { } else { BooleanWritable bw = (BooleanWritable) object; vector[batchIndex] = (bw.get() ? 1 : 0); + colVector.isNull[batchIndex] = false; } } } @@ -156,6 +157,7 @@ void assign(int batchIndex, Object object) { } else { ByteWritable bw = (ByteWritable) object; vector[batchIndex] = bw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -173,6 +175,7 @@ void assign(int batchIndex, Object object) { } else { ShortWritable sw = (ShortWritable) object; vector[batchIndex] = sw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -190,6 +193,7 @@ void assign(int batchIndex, Object object) { } else { IntWritable iw = (IntWritable) object; vector[batchIndex] = iw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -207,6 +211,7 @@ void assign(int batchIndex, Object object) { } else { LongWritable lw = (LongWritable) object; vector[batchIndex] = lw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -224,6 +229,7 @@ void assign(int batchIndex, Object object) { } else { DateWritable bw = (DateWritable) object; vector[batchIndex] = bw.getDays(); + colVector.isNull[batchIndex] = false; } } } @@ -242,6 +248,7 @@ void assign(int batchIndex, Object object) { TimestampWritable tw = (TimestampWritable) object; Timestamp t = tw.getTimestamp(); vector[batchIndex] = TimestampUtils.getTimeNanoSec(t); + colVector.isNull[batchIndex] = false; } } } @@ -260,6 +267,7 @@ void assign(int batchIndex, Object object) { HiveIntervalYearMonthWritable iymw = (HiveIntervalYearMonthWritable) object; HiveIntervalYearMonth iym = iymw.getHiveIntervalYearMonth(); vector[batchIndex] = iym.getTotalMonths(); + colVector.isNull[batchIndex] = false; } } } @@ -278,6 +286,7 @@ void assign(int batchIndex, Object object) { HiveIntervalDayTimeWritable idtw = (HiveIntervalDayTimeWritable) object; HiveIntervalDayTime idt = idtw.getHiveIntervalDayTime(); vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(idt); + colVector.isNull[batchIndex] = false; } } } @@ -317,6 +326,7 @@ void assign(int batchIndex, Object object) { } else { FloatWritable fw = (FloatWritable) object; vector[batchIndex] = fw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -334,6 +344,7 @@ void assign(int batchIndex, Object object) { } else { DoubleWritable dw = (DoubleWritable) object; vector[batchIndex] = dw.get(); + colVector.isNull[batchIndex] = false; } } } @@ -370,6 +381,7 @@ void assign(int batchIndex, Object object) { } else { BytesWritable bw = (BytesWritable) object; colVector.setVal(batchIndex, bw.getBytes(), 0, bw.getLength()); + colVector.isNull[batchIndex] = false; } } } @@ -387,6 +399,7 @@ void assign(int batchIndex, Object object) { } else { Text tw = (Text) object; colVector.setVal(batchIndex, tw.getBytes(), 0, tw.getLength()); + colVector.isNull[batchIndex] = false; } } } @@ -411,6 +424,7 @@ void assign(int batchIndex, Object object) { } byte[] bytes = hiveVarchar.getValue().getBytes(); colVector.setVal(batchIndex, bytes, 0, bytes.length); + colVector.isNull[batchIndex] = false; } } } @@ -437,6 +451,7 @@ void assign(int batchIndex, Object object) { // We store CHAR in vector row batch with padding stripped. byte[] bytes = hiveChar.getStrippedValue().getBytes(); colVector.setVal(batchIndex, bytes, 0, bytes.length); + colVector.isNull[batchIndex] = false; } } } @@ -469,6 +484,7 @@ void assign(int batchIndex, Object object) { } else { colVector.set(batchIndex, (HiveDecimalWritable) object); } + colVector.isNull[batchIndex] = false; } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java index c56903e..6b681b3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java @@ -60,12 +60,14 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa if (inColVector.isRepeating) { if (inColVector.noNulls || !inColVector.isNull[0]) { outColVector.vector[outBatchIndex] = inColVector.vector[0]; + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } } else { if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex]; + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } @@ -87,12 +89,14 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa if (inColVector.isRepeating) { if (inColVector.noNulls || !inColVector.isNull[0]) { outColVector.vector[outBatchIndex] = inColVector.vector[0]; + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } } else { if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex]; + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } @@ -122,12 +126,14 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa if (inColVector.isRepeating) { if (inColVector.noNulls || !inColVector.isNull[0]) { outColVector.setVal(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]); + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } } else { if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { outColVector.setVal(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]); + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } @@ -149,12 +155,14 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa if (inColVector.isRepeating) { if (inColVector.noNulls || !inColVector.isNull[0]) { outColVector.setRef(outBatchIndex, inColVector.vector[0], inColVector.start[0], inColVector.length[0]); + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } } else { if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { outColVector.setRef(outBatchIndex, inColVector.vector[inBatchIndex], inColVector.start[inBatchIndex], inColVector.length[inBatchIndex]); + outColVector.isNull[outBatchIndex] = false; } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } @@ -175,12 +183,14 @@ void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBa if (inColVector.isRepeating) { if (inColVector.noNulls || !inColVector.isNull[0]) { + outColVector.isNull[outBatchIndex] = false; outColVector.set(outBatchIndex, inColVector.vector[0]); } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); } } else { if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) { + outColVector.isNull[outBatchIndex] = false; outColVector.set(outBatchIndex, inColVector.vector[inBatchIndex]); } else { VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index 4d86db6..9b086b8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -102,6 +102,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { boolean value = deserializeRead.readBoolean(); colVector.vector[batchIndex] = (value ? 1 : 0); + colVector.isNull[batchIndex] = false; } } } @@ -121,6 +122,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { byte value = deserializeRead.readByte(); colVector.vector[batchIndex] = (long) value; + colVector.isNull[batchIndex] = false; } } } @@ -140,6 +142,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { short value = deserializeRead.readShort(); colVector.vector[batchIndex] = (long) value; + colVector.isNull[batchIndex] = false; } } } @@ -159,6 +162,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { int value = deserializeRead.readInt(); colVector.vector[batchIndex] = (long) value; + colVector.isNull[batchIndex] = false; } } } @@ -178,6 +182,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { long value = deserializeRead.readLong(); colVector.vector[batchIndex] = value; + colVector.isNull[batchIndex] = false; } } } @@ -200,6 +205,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { deserializeRead.readDate(readDateResults); colVector.vector[batchIndex] = (long) readDateResults.getDays(); + colVector.isNull[batchIndex] = false; } } } @@ -223,6 +229,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readTimestamp(readTimestampResults); Timestamp t = readTimestampResults.getTimestamp(); colVector.vector[batchIndex] = TimestampUtils.getTimeNanoSec(t); + colVector.isNull[batchIndex] = false; } } } @@ -246,6 +253,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readIntervalYearMonth(readIntervalYearMonthResults); HiveIntervalYearMonth hiym = readIntervalYearMonthResults.getHiveIntervalYearMonth(); colVector.vector[batchIndex] = hiym.getTotalMonths(); + colVector.isNull[batchIndex] = false; } } } @@ -269,6 +277,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readIntervalDayTime(readIntervalDayTimeResults); HiveIntervalDayTime hidt = readIntervalDayTimeResults.getHiveIntervalDayTime(); colVector.vector[batchIndex] = DateUtils.getIntervalDayTimeTotalNanos(hidt); + colVector.isNull[batchIndex] = false; } } } @@ -295,6 +304,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { float value = deserializeRead.readFloat(); colVector.vector[batchIndex] = (double) value; + colVector.isNull[batchIndex] = false; } } } @@ -314,6 +324,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } else { double value = deserializeRead.readDouble(); colVector.vector[batchIndex] = value; + colVector.isNull[batchIndex] = false; } } } @@ -344,6 +355,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readString(readStringResults); colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, readStringResults.length); + colVector.isNull[batchIndex] = false; } } } @@ -367,6 +379,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readString(readStringResults); colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, readStringResults.length); + colVector.isNull[batchIndex] = false; } } } @@ -396,6 +409,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes, readStringResults.start, readStringResults.length, charTypeInfo.getLength()); colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength); + colVector.isNull[batchIndex] = false; } } } @@ -425,6 +439,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { int adjustedLength = StringExpr.rightTrimAndTruncate(readStringResults.bytes, readStringResults.start, readStringResults.length, charTypeInfo.getLength()); colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength); + colVector.isNull[batchIndex] = false; } } } @@ -454,6 +469,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { int adjustedLength = StringExpr.truncate(readStringResults.bytes, readStringResults.start, readStringResults.length, varcharTypeInfo.getLength()); colVector.setVal(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength); + colVector.isNull[batchIndex] = false; } } } @@ -483,6 +499,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { int adjustedLength = StringExpr.truncate(readStringResults.bytes, readStringResults.start, readStringResults.length, varcharTypeInfo.getLength()); colVector.setRef(batchIndex, readStringResults.bytes, readStringResults.start, adjustedLength); + colVector.isNull[batchIndex] = false; } } } @@ -506,6 +523,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readBinary(readBinaryResults); colVector.setVal(batchIndex, readBinaryResults.bytes, readBinaryResults.start, readBinaryResults.length); + colVector.isNull[batchIndex] = false; } } } @@ -529,6 +547,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readBinary(readBinaryResults); colVector.setRef(batchIndex, readBinaryResults.bytes, readBinaryResults.start, readBinaryResults.length); + colVector.isNull[batchIndex] = false; } } } @@ -552,6 +571,7 @@ void apply(VectorizedRowBatch batch, int batchIndex) throws IOException { deserializeRead.readHiveDecimal(readDecimalResults); HiveDecimal hiveDecimal = readDecimalResults.getHiveDecimal(); colVector.vector[batchIndex].set(hiveDecimal); + colVector.isNull[batchIndex] = false; } } } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java index c076e6c..a5946d1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.vector; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Random; @@ -60,6 +61,12 @@ void testVectorRowObject(int caseNum, Random r) throws HiveException { batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames); VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); + // junk the destination for the 1st pass + for (ColumnVector cv : batch.cols) { + Arrays.fill(cv.isNull, true); + cv.noNulls = false; + } + VectorAssignRowSameBatch vectorAssignRow = new VectorAssignRowSameBatch(); vectorAssignRow.init(source.typeNames()); vectorAssignRow.setOneBatch(batch); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java index ae4b239..be39b4e 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java @@ -563,6 +563,10 @@ void testVectorDeserializeRow(int caseNum, Random r, SerializationType serializa batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames); VectorizedRowBatch batch = batchContext.createVectorizedRowBatch(); + for (ColumnVector cv : batch.cols) { + Arrays.fill(cv.isNull, true); + } + int fieldCount = source.typeNames().size(); DeserializeRead deserializeRead; SerializeWrite serializeWrite; @@ -592,6 +596,12 @@ void testVectorDeserializeRow(int caseNum, Random r, SerializationType serializa VectorDeserializeRow vectorDeserializeRow = new VectorDeserializeRow(deserializeRead); vectorDeserializeRow.init(); + // junk the destination for the 1st pass + for (ColumnVector cv : batch.cols) { + Arrays.fill(cv.isNull, true); + cv.noNulls = false; + } + VectorExtractRowSameBatch vectorExtractRow = new VectorExtractRowSameBatch(); vectorExtractRow.init(source.typeNames()); vectorExtractRow.setOneBatch(batch); -- 2.4.0