diff --git common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java index a8b7b6d186..d8ec25a2b4 100644 --- common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java +++ common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.common.type; +import java.time.DateTimeException; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -75,11 +76,32 @@ private LocalDateTime localDateTime; - /* Private constructor */ - private Timestamp(LocalDateTime localDateTime) { + private static final LocalDateTime MIN_LOCALDATETIME = + LocalDateTime.parse("0000-01-01 00:00:00", PARSE_FORMATTER); + private static final LocalDateTime MAX_LOCALDATETIME = + LocalDateTime.parse("9999-12-31 23:59:59.999999999", PARSE_FORMATTER); + + /* Private constructor. ASSUME TIMESTAMP IS VALID (use method valid() to check)! */ + private Timestamp(LocalDateTime localDateTime) throws DateTimeException { this.localDateTime = localDateTime != null ? localDateTime : EPOCH; } + private void setLocalDateTime(LocalDateTime localDateTime) { + if (valid(localDateTime)) { + this.localDateTime = localDateTime; + } else { + throw new DateTimeException("Timestamp value " + localDateTime + " out of range"); + } + } + + /** + * Valid range is years 0000-9999, and 0000 converts to 0001. + */ + private static boolean valid(LocalDateTime localDateTime) { + return localDateTime.compareTo(MIN_LOCALDATETIME) >= 0 && + localDateTime.compareTo(MAX_LOCALDATETIME) <= 0; + } + public Timestamp() { this(EPOCH); } @@ -89,7 +111,7 @@ public Timestamp(Timestamp t) { } public void set(Timestamp t) { - this.localDateTime = t != null ? t.localDateTime : EPOCH; + setLocalDateTime(t != null ? t.localDateTime : EPOCH); } public String format(DateTimeFormatter formatter) { @@ -122,28 +144,26 @@ public long toEpochSecond() { return localDateTime.toEpochSecond(ZoneOffset.UTC); } - public void setTimeInSeconds(long epochSecond) { + public void setTimeInSeconds(long epochSecond) throws DateTimeException { setTimeInSeconds(epochSecond, 0); } - public void setTimeInSeconds(long epochSecond, int nanos) { - localDateTime = LocalDateTime.ofEpochSecond( - epochSecond, nanos, ZoneOffset.UTC); + public void setTimeInSeconds(long epochSecond, int nanos) throws DateTimeException { + setLocalDateTime(LocalDateTime.ofEpochSecond(epochSecond, nanos, ZoneOffset.UTC)); } public long toEpochMilli() { return localDateTime.toInstant(ZoneOffset.UTC).toEpochMilli(); } - public void setTimeInMillis(long epochMilli) { - localDateTime = LocalDateTime.ofInstant( - Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC); + public void setTimeInMillis(long epochMilli) throws DateTimeException { + setLocalDateTime(LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC)); } - public void setTimeInMillis(long epochMilli, int nanos) { - localDateTime = LocalDateTime + public void setTimeInMillis(long epochMilli, int nanos) throws DateTimeException { + setLocalDateTime(LocalDateTime .ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC) - .withNano(nanos); + .withNano(nanos)); } public int getNanos() { @@ -163,7 +183,10 @@ public static Timestamp valueOf(String s) { throw new IllegalArgumentException("Cannot create timestamp, parsing error"); } } - return new Timestamp(localDateTime); + if (valid(localDateTime)) { + return new Timestamp(localDateTime); + } + return null; } public static Timestamp ofEpochSecond(long epochSecond) { @@ -171,23 +194,33 @@ public static Timestamp ofEpochSecond(long epochSecond) { } public static Timestamp ofEpochSecond(long epochSecond, int nanos) { - return new Timestamp( - LocalDateTime.ofEpochSecond(epochSecond, nanos, ZoneOffset.UTC)); + LocalDateTime localDateTime = LocalDateTime.ofEpochSecond(epochSecond, nanos, ZoneOffset.UTC); + if (valid(localDateTime)) { + return new Timestamp(localDateTime); + } + return null; } public static Timestamp ofEpochMilli(long epochMilli) { - return new Timestamp(LocalDateTime - .ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC)); + LocalDateTime localDateTime = + LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC); + if (valid(localDateTime)) { + return new Timestamp(localDateTime); + } + return null; } public static Timestamp ofEpochMilli(long epochMilli, int nanos) { - return new Timestamp(LocalDateTime - .ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC) - .withNano(nanos)); + LocalDateTime localDateTime = + LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneOffset.UTC).withNano(nanos); + if (valid(localDateTime)) { + return new Timestamp(localDateTime); + } + return null; } public void setNanos(int nanos) { - localDateTime = localDateTime.withNano(nanos); + setLocalDateTime(localDateTime.withNano(nanos)); } public int getYear() { diff --git common/src/java/org/apache/hadoop/hive/common/type/TimestampUtils.java common/src/java/org/apache/hadoop/hive/common/type/TimestampUtils.java index f26f8ae01e..b554b55da7 100644 --- common/src/java/org/apache/hadoop/hive/common/type/TimestampUtils.java +++ common/src/java/org/apache/hadoop/hive/common/type/TimestampUtils.java @@ -156,6 +156,23 @@ public static Timestamp decimalToTimestamp(HiveDecimalV1 dec) { return null; } } + + /** + * Converts the time in seconds or milliseconds to a timestamp. + * @param time time in seconds or in milliseconds + * @return the timestamp + */ + public static Timestamp longToTimestamp(long time, boolean intToTimestampInSeconds) { + try { + // If the time is in seconds, converts it to milliseconds first. + if (intToTimestampInSeconds) { + return Timestamp.ofEpochSecond(time); + } + return Timestamp.ofEpochMilli(time); + } catch (DateTimeException e) { + return null; + } + } /** * Rounds the number of milliseconds relative to the epoch down to the nearest whole number of @@ -186,9 +203,26 @@ public static Timestamp stringToTimestamp(String s) { return Timestamp.valueOf( TimestampTZUtil.parse(s).getZonedDateTime().toLocalDateTime().toString()); } catch (IllegalArgumentException | DateTimeParseException eTZ) { - // Last attempt - return Timestamp.ofEpochMilli(Date.valueOf(s).toEpochMilli()); + try { + // Last attempt + return Timestamp.ofEpochMilli(Date.valueOf(s).toEpochMilli()); + } catch (IllegalArgumentException | DateTimeParseException | NullPointerException eTZ2) { + return null; + } } } } + + public static Timestamp timestampLocalTzToTimestamp(String tstz) { + int index = tstz.indexOf(" "); + index = tstz.indexOf(" ", index + 1); + if (index == -1) { + return null; + } + try { + return Timestamp.valueOf(tstz.substring(0, index)); + } catch (DateTimeException e) { + return null; + } + } } diff --git common/src/test/org/apache/hive/common/util/TestTimestampParser.java common/src/test/org/apache/hive/common/util/TestTimestampParser.java index 00a7904ecf..504ddc8842 100644 --- common/src/test/org/apache/hive/common/util/TestTimestampParser.java +++ common/src/test/org/apache/hive/common/util/TestTimestampParser.java @@ -19,6 +19,7 @@ package org.apache.hive.common.util; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; import org.apache.hadoop.hive.common.type.Timestamp; @@ -53,6 +54,13 @@ static void testInvalidCases(TimestampParser tp, String[] invalidCases) { } } + static void testOutOfRangeCases(TimestampParser tp, String[] outOfRangeCases) { + for (String outOfRangeString : outOfRangeCases) { + Timestamp ts = tp.parseTimestamp(outOfRangeString); + assertNull(ts); + } + } + @Test public void testDefault() { // No timestamp patterns, should default to normal timestamp format @@ -72,8 +80,14 @@ public void testDefault() { "12345", }; + String[] outOfRangeCases = { + "-0001-02-03 04:05:06.7890", + "10000-01-01 00:00:00.000", + }; + testValidCases(tp, validCases); testInvalidCases(tp, invalidCases); + testOutOfRangeCases(tp, outOfRangeCases); } @Test @@ -120,8 +134,14 @@ public void testPattern1() { "12345", }; + String[] outOfRangeCases = { + "-0001-12-31 23:59:59.999999999", + "10000-01-01 00:00:00.000", + }; + testValidCases(tp, validCases); testInvalidCases(tp, invalidCases); + testOutOfRangeCases(tp, outOfRangeCases); } @Test @@ -151,8 +171,14 @@ public void testMillisParser() { "1420509274123-", }; + String[] outOfRangeCases = { + "-0001-12-31 23:59:59.999999999", + "10000-01-01 00:00:00.000", + }; + testValidCases(tp, validCases); testInvalidCases(tp, invalidCases); + testOutOfRangeCases(tp, outOfRangeCases); } @Test @@ -178,7 +204,13 @@ public void testPattern2() { "12345", }; + String[] outOfRangeCases = { + "-0001-12-31 23:59:59.999999999", + "10000-01-01 00:00:00.000", + }; + testValidCases(tp, validCases); testInvalidCases(tp, invalidCases); + testOutOfRangeCases(tp, outOfRangeCases); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java index 20acfa253c..1ccdb7cc66 100644 --- ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java +++ ql/src/java/org/apache/hadoop/hive/ql/util/DateTimeMath.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hive.common.util.DateUtils; +import java.time.DateTimeException; import java.util.Calendar; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -108,7 +109,12 @@ public boolean add(Timestamp ts, HiveIntervalYearMonth interval, Timestamp resul } long resultMillis = addMonthsToMillis(ts.toEpochMilli(), interval.getTotalMonths()); - result.setTimeInMillis(resultMillis, ts.getNanos()); + try { + result.setTimeInMillis(resultMillis, ts.getNanos()); + } catch (DateTimeException e) { + result = null; + return false; + } return true; } @@ -157,7 +163,12 @@ public boolean add(HiveIntervalYearMonth interval, Timestamp ts, Timestamp resul } long resultMillis = addMonthsToMillis(ts.toEpochMilli(), interval.getTotalMonths()); - result.setTimeInMillis(resultMillis, ts.getNanos()); + try { + result.setTimeInMillis(resultMillis, ts.getNanos()); + } catch (DateTimeException e) { + result = null; + return false; + } return true; } @@ -409,7 +420,12 @@ public boolean add(Timestamp ts, HiveIntervalDayTime interval, long newMillis = ts.toEpochMilli() + TimeUnit.SECONDS.toMillis(interval.getTotalSeconds() + nanosResult.seconds); - result.setTimeInMillis(newMillis, nanosResult.nanos); + try { + result.setTimeInMillis(newMillis, nanosResult.nanos); + } catch (DateTimeException e) { + result = null; + return false; + } return true; } @@ -460,7 +476,12 @@ public boolean add(HiveIntervalDayTime interval, Timestamp ts, long newMillis = ts.toEpochMilli() + TimeUnit.SECONDS.toMillis(interval.getTotalSeconds() + nanosResult.seconds); - result.setTimeInMillis(newMillis, nanosResult.nanos); + try { + result.setTimeInMillis(newMillis, nanosResult.nanos); + } catch (DateTimeException e) { + result = null; + return false; + } return true; } diff --git ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java index 2e011b58cb..6e21b19213 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/arrow/TestArrowColumnarBatchSerDe.java @@ -59,6 +59,7 @@ import org.junit.Before; import org.junit.Test; +import java.time.DateTimeException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -510,7 +511,13 @@ public void testPrimitiveRandomTimestamp() throws SerDeException { for (int i = 0; i < size; i++) { long millis = ((long) rand.nextInt(Integer.MAX_VALUE)) * 1000; Timestamp timestamp = Timestamp.ofEpochMilli(rand.nextBoolean() ? millis : -millis); - timestamp.setNanos(rand.nextInt(1000) * 1000); + if (timestamp != null) { + try { + timestamp.setNanos(rand.nextInt(1000) * 1000); + } catch (DateTimeException e) { + timestamp = null; + } + } rows[i] = new Object[] {new TimestampWritableV2(timestamp)}; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritableV2.java serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritableV2.java index 9aa7f19ab2..927dc5da8a 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritableV2.java +++ serde/src/java/org/apache/hadoop/hive/serde2/io/TimestampWritableV2.java @@ -20,6 +20,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.time.DateTimeException; import java.time.format.DateTimeFormatter; import org.apache.hadoop.hive.common.type.HiveDecimal; @@ -113,10 +114,10 @@ public void set(byte[] bytes, int offset) { public void set(Timestamp t) { if (t == null) { - timestamp.set(null); - return; + timestamp = null; + } else { + timestamp.set(t); } - timestamp.set(t); bytesEmpty = true; timestampEmpty = false; } @@ -134,7 +135,11 @@ public void set(TimestampWritableV2 t) { } public static void updateTimestamp(Timestamp timestamp, long secondsAsMillis, int nanos) { - timestamp.setTimeInMillis(secondsAsMillis, nanos); + try { + timestamp.setTimeInMillis(secondsAsMillis, nanos); + } catch (DateTimeException e) { + timestamp = null; + } } public void setInternal(long secondsAsMillis, int nanos) { @@ -366,6 +371,9 @@ public boolean equals(Object o) { @Override public String toString() { + if (timestamp == null) { + return "NULL"; + } if (timestampEmpty) { populateTimestamp(); } @@ -398,7 +406,11 @@ public int hashCode() { private void populateTimestamp() { long seconds = getSeconds(); int nanos = getNanos(); - timestamp.setTimeInSeconds(seconds, nanos); + try { + timestamp.setTimeInSeconds(seconds, nanos); + } catch (DateTimeException e) { + timestamp = null; + } } /** Static methods **/ @@ -523,19 +535,6 @@ public static HiveDecimal getHiveDecimal(Timestamp timestamp) { return result; } - /** - * Converts the time in seconds or milliseconds to a timestamp. - * @param time time in seconds or in milliseconds - * @return the timestamp - */ - public static Timestamp longToTimestamp(long time, boolean intToTimestampInSeconds) { - // If the time is in seconds, converts it to milliseconds first. - if (intToTimestampInSeconds) { - return Timestamp.ofEpochSecond(time); - } - return Timestamp.ofEpochMilli(time); - } - public static void setTimestamp(Timestamp t, byte[] bytes, int offset) { long seconds = getSeconds(bytes, offset); int nanos; @@ -544,7 +543,11 @@ public static void setTimestamp(Timestamp t, byte[] bytes, int offset) { } else { nanos = 0; } - t.setTimeInSeconds(seconds, nanos); + try { + t.setTimeInSeconds(seconds, nanos); + } catch (DateTimeException e) { + t = null; + } } public static Timestamp createTimestamp(byte[] bytes, int offset) { diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java index 47719c8564..dbbb5137b1 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java @@ -21,6 +21,8 @@ import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import java.time.DateTimeException; + public class JavaTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector implements SettableTimestampObjectInspector { @@ -56,7 +58,11 @@ public Object set(Object o, java.sql.Timestamp value) { if (value == null) { return null; } - ((Timestamp) o).setTimeInMillis(value.getTime(), value.getNanos()); + try { + ((Timestamp) o).setTimeInMillis(value.getTime(), value.getNanos()); + } catch (DateTimeException e) { + o = null; + } return o; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 3886b202c7..e383c6a62e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -1194,23 +1194,23 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, break; case BOOLEAN: longValue = ((BooleanObjectInspector) inputOI).get(o) ? 1 : 0; - result = TimestampWritableV2.longToTimestamp(longValue, intToTimestampInSeconds); + result = TimestampUtils.longToTimestamp(longValue, intToTimestampInSeconds); break; case BYTE: longValue = ((ByteObjectInspector) inputOI).get(o); - result = TimestampWritableV2.longToTimestamp(longValue, intToTimestampInSeconds); + result = TimestampUtils.longToTimestamp(longValue, intToTimestampInSeconds); break; case SHORT: longValue = ((ShortObjectInspector) inputOI).get(o); - result = TimestampWritableV2.longToTimestamp(longValue, intToTimestampInSeconds); + result = TimestampUtils.longToTimestamp(longValue, intToTimestampInSeconds); break; case INT: longValue = ((IntObjectInspector) inputOI).get(o); - result = TimestampWritableV2.longToTimestamp(longValue, intToTimestampInSeconds); + result = TimestampUtils.longToTimestamp(longValue, intToTimestampInSeconds); break; case LONG: longValue = ((LongObjectInspector) inputOI).get(o); - result = TimestampWritableV2.longToTimestamp(longValue, intToTimestampInSeconds); + result = TimestampUtils.longToTimestamp(longValue, intToTimestampInSeconds); break; case FLOAT: result = TimestampUtils.doubleToTimestamp(((FloatObjectInspector) inputOI).get(o)); @@ -1239,13 +1239,7 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, result = ((TimestampObjectInspector) inputOI).getPrimitiveWritableObject(o).getTimestamp(); break; case TIMESTAMPLOCALTZ: - String tstz = inputOI.getPrimitiveWritableObject(o).toString(); - int index = tstz.indexOf(" "); - index = tstz.indexOf(" ", index + 1); - if (index == -1) { - return null; - } - result = Timestamp.valueOf(tstz.substring(0, index)); + result = TimestampUtils.timestampLocalTzToTimestamp(inputOI.getPrimitiveWritableObject(o).toString()); break; default: throw new RuntimeException("Hive 2 Internal error: unknown type: " diff --git serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritableV2.java serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritableV2.java index 155dc1f58c..3378fc6f03 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritableV2.java +++ serde/src/test/org/apache/hadoop/hive/serde2/io/TestTimestampWritableV2.java @@ -247,6 +247,9 @@ private static int randomNanos(Random rand) { private static void checkTimestampWithAndWithoutNanos(Timestamp ts, int nanos) throws IOException { + if (ts == null) { + return; + } serializeDeserializeAndCheckTimestamp(ts); ts.setNanos(nanos); @@ -475,7 +478,9 @@ public void testBinarySortable() { List tswList = new ArrayList(); for (int i = 0; i < 50; ++i) { Timestamp ts = Timestamp.ofEpochMilli(rand.nextLong(), randomNanos(rand)); - tswList.add(new TimestampWritableV2(ts)); + if (ts != null) { + tswList.add(new TimestampWritableV2(ts)); + } } for (TimestampWritableV2 tsw1 : tswList) { byte[] bs1 = tsw1.getBinarySortable();