diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 173fb8d..0876bf7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -1018,6 +1018,23 @@ Object next(Object previous) throws IOException { } @Override + Object nextVector(Object previousVector, long batchSize) throws IOException { + LongColumnVector result = null; + if (previousVector == null) { + result = new LongColumnVector(); + } else { + result = (LongColumnVector) previousVector; + } + + // Read present/isNull stream + super.nextVector(result, batchSize); + + // Read value entries based on isNull entries + reader.nextVector(result, batchSize); + return result; + } + + @Override void skipRows(long items) throws IOException { reader.skip(countNonNulls(items)); } diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java index ef9b968..0d5b7ff 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java @@ -18,24 +18,26 @@ package org.apache.hadoop.hive.ql.io.orc; -import java.io.File; -import java.sql.Timestamp; -import java.util.Calendar; -import java.util.Random; - import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.io.BooleanWritable; +import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.junit.Before; import org.junit.Test; +import java.io.File; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.Calendar; +import java.util.Random; + /** * * Class that tests ORC reader vectorization by comparing records that are @@ -68,8 +70,10 @@ public void openFileSystem() throws Exception { private final Double d; private final String k; private final Timestamp t; + private final Date dt; - MyRecord(Boolean bo, Byte by, Integer i, Long l, Short s, Double d, String k, Timestamp t) { + MyRecord(Boolean bo, Byte by, Integer i, Long l, Short s, Double d, String k, Timestamp t, + Date dt) { this.bo = bo; this.by = by; this.i = i; @@ -78,6 +82,7 @@ public void openFileSystem() throws Exception { this.d = d; this.k = k; this.t = t; + this.dt = dt; } } @@ -103,12 +108,14 @@ public void createFile() throws Exception { "before", "us,", "we", "were", "all", "going", "direct", "to", "Heaven,", "we", "were", "all", "going", "direct", "the", "other", "way"}; + String[] dates = new String[] {"1991-02-28", "1970-01-31", "1950-04-23"}; for (int i = 0; i < 21000; ++i) { if ((i % 7) != 0) { writer.addRow(new MyRecord(((i % 3) == 0), (byte)(i % 5), i, (long) 200, (short) (300 + i), (double) (400 + i), - words[r1.nextInt(words.length)], new Timestamp(Calendar.getInstance().getTime().getTime()))); + words[r1.nextInt(words.length)], new Timestamp(Calendar.getInstance().getTime().getTime()), + Date.valueOf(dates[i % 3]))); } else { - writer.addRow(new MyRecord(null, null, i, (long) 200, null, null, null, null)); + writer.addRow(new MyRecord(null, null, i, (long) 200, null, null, null, null, null)); } } writer.close(); @@ -148,6 +155,13 @@ private void checkVectorizedReader() throws Exception { Assert.assertEquals(true, timeInNanoSec.toString().equals(b.toString())); continue; } + + // Dates are stored as long, so convert and compare + if (a instanceof Date) { + Date adt = (Date) a; + Assert.assertEquals(adt.getTime(), DateWritable.daysToMillis((int) ((LongWritable) b).get())); + continue; + } if (null == a) { Assert.assertEquals(true, (b == null || (b instanceof NullWritable))); } else { @@ -165,6 +179,7 @@ private void checkVectorizedReader() throws Exception { Assert.assertEquals(false, batch.cols[5].isRepeating); Assert.assertEquals(false, batch.cols[6].isRepeating); Assert.assertEquals(false, batch.cols[7].isRepeating); + Assert.assertEquals(false, batch.cols[8].isRepeating); // Check non null Assert.assertEquals(false, batch.cols[0].noNulls); @@ -175,6 +190,7 @@ private void checkVectorizedReader() throws Exception { Assert.assertEquals(false, batch.cols[5].noNulls); Assert.assertEquals(false, batch.cols[6].noNulls); Assert.assertEquals(false, batch.cols[7].noNulls); + Assert.assertEquals(false, batch.cols[8].noNulls); } Assert.assertEquals(false, rr.hasNext()); }