commit 81f05f07fc0dfcf8fd75adaa2b3b61c2aaccbcb9 Author: Bharath Krishna Date: Tue May 1 11:13:34 2018 -0700 HIVE-19354 : from_utc_timestamp returns incorrect results for datetime values with timezone diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java index 8691ed15e3033fe5a2e22c858c0c0bf81640b2ad..a58ab9eb990ad12daecaf7b6a6289d9245b02456 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java @@ -20,6 +20,8 @@ import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.util.Date; import java.util.TimeZone; import org.slf4j.Logger; @@ -41,7 +43,7 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF { static final Logger LOG = LoggerFactory.getLogger(GenericUDFFromUtcTimestamp.class); - + private static final String UTC_ZONE_ID = "UTC"; private transient PrimitiveObjectInspector[] argumentOIs; private transient TimestampConverter timestampConverter; private transient TextConverter textConverter; @@ -101,7 +103,7 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { return null; } - Object converted_o0 = timestampConverter.convert(o0); + Object converted_o0 = timestampConverter.convert(o0, ZoneId.of(UTC_ZONE_ID)); if (converted_o0 == null) { return null; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java index ba20a2c56d0d45e37fedd14f19e4e73000b5544f..b6c5063f0e24d761d2fd801aef56612854d614b8 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorConverter.java @@ -283,13 +283,16 @@ public TimestampConverter(PrimitiveObjectInspector inputOI, public void setIntToTimestampInSeconds(boolean intToTimestampInSeconds) { this.intToTimestampInSeconds = intToTimestampInSeconds; } - + public Object convert(Object input) { + return convert(input, null); + } + public Object convert(Object input, ZoneId zoneId) { if (input == null) { return null; } return outputOI.set(r, PrimitiveObjectInspectorUtils.getTimestamp(input, - inputOI, intToTimestampInSeconds)); + inputOI, intToTimestampInSeconds, zoneId)); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 8cf074452976aba0d98cd92c94448aa073f1a7b7..e54ebc18a6c9ea8e3be6281e196a63b33b2568d4 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -27,6 +27,7 @@ import java.sql.Timestamp; import java.time.DateTimeException; import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.HashMap; import java.util.Map; @@ -1171,10 +1172,14 @@ public static Date getDate(Object o, PrimitiveObjectInspector oi) { } public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector oi) { - return getTimestamp(o, oi, false); + return getTimestamp(o, oi, false, null); } - public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, boolean intToTimestampInSeconds) { + return getTimestamp(o, inputOI, intToTimestampInSeconds, null); + } + + public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, + boolean intToTimestampInSeconds, ZoneId zoneId) { if (o == null) { return null; } @@ -1218,11 +1223,11 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, case STRING: StringObjectInspector soi = (StringObjectInspector) inputOI; String s = soi.getPrimitiveJavaObject(o); - result = getTimestampFromString(s); + result = getTimestampFromString(s, zoneId); break; case CHAR: case VARCHAR: - result = getTimestampFromString(getString(o, inputOI)); + result = getTimestampFromString(getString(o, inputOI), zoneId); break; case DATE: result = new Timestamp( @@ -1248,6 +1253,9 @@ public static Timestamp getTimestamp(Object o, PrimitiveObjectInspector inputOI, } static Timestamp getTimestampFromString(String s) { + return getTimestampFromString(s, null); + } + static Timestamp getTimestampFromString(String s, ZoneId zoneId) { Timestamp result; s = s.trim(); s = trimNanoTimestamp(s); @@ -1261,7 +1269,15 @@ static Timestamp getTimestampFromString(String s) { } catch (IllegalArgumentException e) { // Let's try to parse it as timestamp with time zone and transform try { - result = Timestamp.from(TimestampTZUtil.parse(s).getZonedDateTime().toInstant()); + if (zoneId == null) { + result = Timestamp.from(TimestampTZUtil.parse(s).getZonedDateTime().toInstant()); + } else { + //Convert to timestamp according to the zoneId provided + result = Timestamp.valueOf(ZonedDateTime + .ofInstant(TimestampTZUtil.parse(s).getZonedDateTime().toInstant(), zoneId) + .toLocalDateTime()); + } + } catch (DateTimeException e2) { result = null; }