diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java index a4497b3..3b684d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthIntegerWriterV2.java @@ -562,7 +562,7 @@ private void determineEncoding() { private void preparePatchedBlob() { // mask will be max value beyond which patch will be generated - int mask = (1 << brBits95p) - 1; + long mask = (1L << brBits95p) - 1; // since we are considering only 95 percentile, the size of gap and // patch array can contain only be 5% values diff --git ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java index 6b5178a..5919199 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java @@ -25,6 +25,7 @@ import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.Charset; +import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -46,6 +47,14 @@ public class TestNewIntegerEncoding { + public static class TSRow { + Timestamp ts; + + public TSRow(Timestamp ts) { + this.ts = ts; + } + } + public static class Row { Integer int1; Long long1; @@ -878,6 +887,68 @@ public void testPatchedBase511() throws Exception { } @Test + public void testPatchedBaseTimestamp() throws Exception { + ObjectInspector inspector; + synchronized (TestOrcFile.class) { + inspector = ObjectInspectorFactory.getReflectionObjectInspector(TSRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + + Writer writer = OrcFile.createWriter(testFilePath, + OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000).bufferSize(10000)); + + List tslist = Lists.newArrayList(); + tslist.add(Timestamp.valueOf("9999-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2003-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("1999-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("1995-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2002-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2010-03-02 00:00:00")); + tslist.add(Timestamp.valueOf("2005-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2006-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2003-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("1996-08-02 00:00:00")); + tslist.add(Timestamp.valueOf("1998-11-02 00:00:00")); + tslist.add(Timestamp.valueOf("2008-10-02 00:00:00")); + tslist.add(Timestamp.valueOf("1993-08-02 00:00:00")); + tslist.add(Timestamp.valueOf("2008-01-02 00:00:00")); + tslist.add(Timestamp.valueOf("2007-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2004-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2008-10-02 00:00:00")); + tslist.add(Timestamp.valueOf("2003-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2004-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2008-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2005-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("1994-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2006-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2004-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2001-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2000-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2000-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2002-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2006-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2011-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2002-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("2005-01-01 00:00:00")); + tslist.add(Timestamp.valueOf("1974-01-01 00:00:00")); + + for (Timestamp ts : tslist) { + writer.addRow(new TSRow(ts)); + } + + writer.close(); + + Reader reader = OrcFile.createReader(fs, testFilePath); + RecordReader rows = reader.rows(null); + int idx = 0; + while (rows.hasNext()) { + Object row = rows.next(null); + assertEquals(tslist.get(idx++).getNanos(), + ((Timestamp) ((OrcStruct) row).getFieldValue(0)).getNanos()); + } + } + + @Test public void testSeek() throws Exception { ObjectInspector inspector; synchronized (TestOrcFile.class) {