diff --git ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java index c5079e1107..5ec800d1e6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java @@ -816,8 +816,11 @@ private static void writeGeneric(final FieldVector fieldVector, final ColumnVect final TimestampColumnVector timestampColumnVector = (TimestampColumnVector) hiveVector; // Time = second + sub-second final long secondInMillis = timestampColumnVector.getTime(j); - final long secondInMicros = (secondInMillis - secondInMillis % MILLIS_PER_SECOND) * MICROS_PER_MILLIS; - final long subSecondInMicros = timestampColumnVector.getNanos(j) / NS_PER_MICROS; + final long nanos = timestampColumnVector.getNanos(j); + // nanos should be subtracted from total time(secondInMillis) to obtain micros + // Divide nanos by 1000_000 to bring it to millisecond precision and then perform the subtraction + final long secondInMicros = (secondInMillis - nanos / NS_PER_MILLIS) * MICROS_PER_MILLIS; + final long subSecondInMicros = nanos / NS_PER_MICROS; if ((secondInMillis > 0 && secondInMicros < 0) || (secondInMillis < 0 && secondInMicros > 0)) { // If the timestamp cannot be represented in long microsecond, set it as a null value timeStampMicroTZVector.setNull(i); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java index 2e011b58cb..f1cb6dfea7 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java @@ -517,6 +517,45 @@ public void testPrimitiveRandomTimestamp() throws SerDeException { initAndSerializeAndDeserialize(schema, rows); } + + @Test + public void testTimestampNanosPrecisionUpTo6Digits() throws SerDeException { + String[][] schema = { + {"timestamp1", "timestamp"}, + }; + //Nanos precise upto 6 digits + Object[][] tsRows = new Object[][]{ + {new TimestampWritableV2(Timestamp.valueOf("1800-04-01 09:01:10.123999"))}, + {new TimestampWritableV2(Timestamp.valueOf("2050-04-01 09:01:10.999999"))}, + null + }; + initAndSerializeAndDeserialize(schema, tsRows); + } + + @Test + public void testPositiveNegativeTSWithNanos() throws SerDeException { + String[][] schema = { + {"timestamp1", "timestamp"}, + }; + + Object[][] tsRows = new Object[][]{ + {new TimestampWritableV2(Timestamp.valueOf("1963-04-01 09:01:10.123"))}, + {new TimestampWritableV2(Timestamp.valueOf("1800-04-01 09:01:10.123999"))}, + {new TimestampWritableV2(Timestamp.valueOf("1750-04-01 09:01:10.123999"))}, + {new TimestampWritableV2(Timestamp.valueOf("1700-04-01 09:01:10.999999"))}, + {new TimestampWritableV2(Timestamp.valueOf("2050-04-01 09:01:10.999999"))}, + {new TimestampWritableV2(Timestamp.valueOf("1991-06-05 09:01:10.999999"))}, + {new TimestampWritableV2(Timestamp.valueOf("1992-11-04 09:01:10.999999"))}, + {new TimestampWritableV2(Timestamp.valueOf("1970-01-01 00:00:00"))}, + {new TimestampWritableV2(Timestamp.valueOf("1964-01-01 00:00:04.78"))}, + {new TimestampWritableV2(Timestamp.valueOf("1950-01-01 09:23:03.21"))}, + {new TimestampWritableV2(Timestamp.valueOf("1956-01-01 10:09:03.00"))}, + {new TimestampWritableV2(Timestamp.valueOf("1947-08-27 10:25:36.26"))}, + null + }; + initAndSerializeAndDeserialize(schema, tsRows); + } + @Test public void testPrimitiveDecimal() throws SerDeException { String[][] schema = {