diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 64e7e0a..d407c3a 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -885,7 +885,9 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { "Maximum fraction of heap that can be used by Parquet file writers in one task.\n" + "It is for avoiding OutOfMemory error in tasks. Work with Parquet 1.6.0 and above.\n" + "This config parameter is defined in Parquet, so that it does not start with 'hive.'."), - + HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION("hive.parquet.timestamp.skip.conversion", true, + "Current Hive implementation of parquet stores timestamps to UTC, this flag allows skipping of the conversion" + + "on reading parquet files from other tools"), HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f, "Maximum fraction of heap that can be used by ORC file writers"), HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null, diff --git data/files/parquet_external_time.parq data/files/parquet_external_time.parq new file mode 100644 index 0000000..b4d9f43 Binary files /dev/null and data/files/parquet_external_time.parq differ diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ConverterParent.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ConverterParent.java index a86d6f4..6ff6b47 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ConverterParent.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ConverterParent.java @@ -1,7 +1,24 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.hive.ql.io.parquet.convert; import org.apache.hadoop.io.Writable; +import java.util.Map; + interface ConverterParent { void set(int index, Writable value); + + Map getMetadata(); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java index 000e8ea..a43661e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/DataWritableRecordConverter.java @@ -13,10 +13,15 @@ */ package org.apache.hadoop.hive.ql.io.parquet.convert; +import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; import org.apache.hadoop.io.ArrayWritable; import parquet.io.api.GroupConverter; import parquet.io.api.RecordMaterializer; import parquet.schema.GroupType; +import parquet.schema.MessageType; +import parquet.schema.MessageTypeParser; + +import java.util.Map; /** * @@ -27,8 +32,9 @@ private final HiveStructConverter root; - public DataWritableRecordConverter(final GroupType requestedSchema, final GroupType tableSchema) { - this.root = new HiveStructConverter(requestedSchema, tableSchema); + public DataWritableRecordConverter(final GroupType requestedSchema, final Map metadata) { + this.root = new HiveStructConverter(requestedSchema, + MessageTypeParser.parseMessageType(metadata.get(DataWritableReadSupport.HIVE_SCHEMA_KEY)), metadata); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java index 23bb364..377e362 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java @@ -16,7 +16,10 @@ import java.math.BigDecimal; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Map; +import com.google.common.base.Strings; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime; import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; import org.apache.hadoop.hive.serde2.io.DoubleWritable; @@ -32,7 +35,6 @@ import parquet.column.Dictionary; import parquet.io.api.Binary; -import parquet.io.api.Converter; import parquet.io.api.PrimitiveConverter; import parquet.schema.OriginalType; import parquet.schema.PrimitiveType; @@ -140,7 +142,14 @@ PrimitiveConverter getConverter(final PrimitiveType type, final int index, final @Override protected TimestampWritable convert(Binary binary) { NanoTime nt = NanoTime.fromBinary(binary); - Timestamp ts = NanoTimeUtils.getTimestamp(nt); + Map metadata = parent.getMetadata(); + //Current Hive parquet timestamp implementation stores it in UTC, but other components do not do that. + //If this file written by current Hive implementation itself, we need to do the reverse conversion, else skip the conversion. + boolean skipConversion = false; + if (Boolean.valueOf(metadata.get(HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION.varname))) { + skipConversion = !Strings.nullToEmpty(metadata.get("createdBy")).startsWith("parquet-mr"); + } + Timestamp ts = NanoTimeUtils.getTimestamp(nt, skipConversion); return new TimestampWritable(ts); } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java index 872900b..6621a87 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java @@ -3,6 +3,8 @@ import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.List; +import java.util.Map; + import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; import parquet.io.api.Converter; @@ -27,12 +29,13 @@ public static HiveGroupConverter forList(GroupType listType, ConverterParent parent, int index) { return new HiveCollectionConverter( - listType, parent, index, false /* not a map */ ); + listType, parent, index, false /* not a map */); } private HiveCollectionConverter(GroupType collectionType, ConverterParent parent, int index, boolean isMap) { + setMetadata(parent.getMetadata()); this.collectionType = collectionType; this.parent = parent; this.index = index; @@ -78,6 +81,7 @@ public void set(int index, Writable value) { private Writable[] keyValue = null; public KeyValueConverter(GroupType keyValueType, HiveGroupConverter parent) { + setMetadata(parent.getMetadata()); this.parent = parent; this.keyConverter = getConverterFromDescription( keyValueType.getType(0), 0, this); @@ -120,6 +124,7 @@ public void end() { private Writable element = null; public ElementConverter(GroupType repeatedType, HiveGroupConverter parent) { + setMetadata(parent.getMetadata()); this.parent = parent; this.elementConverter = getConverterFromDescription( repeatedType.getType(0), 0, this); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java index 11772be..d0dae2b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveGroupConverter.java @@ -23,8 +23,20 @@ import parquet.schema.PrimitiveType; import parquet.schema.Type; +import java.util.Map; + public abstract class HiveGroupConverter extends GroupConverter implements ConverterParent { + private Map metadata; + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getMetadata() { + return metadata; + } + protected static PrimitiveConverter getConverterFromDescription(PrimitiveType type, int index, ConverterParent parent) { if (type == null) { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java index eeb3838..f95d15e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveStructConverter.java @@ -15,6 +15,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; + import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; import parquet.io.api.Converter; @@ -36,8 +38,9 @@ private final List repeatedConverters; private boolean reuseWritableArray = false; - public HiveStructConverter(final GroupType requestedSchema, final GroupType tableSchema) { + public HiveStructConverter(final GroupType requestedSchema, final GroupType tableSchema, Map metadata) { this(requestedSchema, null, 0, tableSchema); + setMetadata(metadata); this.reuseWritableArray = true; this.writables = new Writable[tableSchema.getFieldCount()]; } @@ -49,6 +52,9 @@ public HiveStructConverter(final GroupType groupType, final ConverterParent pare public HiveStructConverter(final GroupType selectedGroupType, final ConverterParent parent, final int index, final GroupType containingGroupType) { + if (parent != null) { + setMetadata(parent.getMetadata()); + } this.parent = parent; this.index = index; this.totalFieldCount = containingGroupType.getFieldCount(); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java index af28b4c..fdea782 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java @@ -1,7 +1,10 @@ package org.apache.hadoop.hive.ql.io.parquet.convert; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; + import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; import parquet.column.Dictionary; @@ -21,11 +24,23 @@ public void parentEnd(); + abstract class RepeatedConverterParent extends PrimitiveConverter implements Repeated { + private Map metadata; + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Map getMetadata() { + return metadata; + } + } + /** * Stands in for a PrimitiveConverter and accumulates multiple values as an * ArrayWritable. */ - class RepeatedPrimitiveConverter extends PrimitiveConverter implements Repeated { + class RepeatedPrimitiveConverter extends RepeatedConverterParent { private final PrimitiveType primitiveType; private final PrimitiveConverter wrapped; private final ConverterParent parent; @@ -33,6 +48,7 @@ private final List list = new ArrayList(); public RepeatedPrimitiveConverter(PrimitiveType primitiveType, ConverterParent parent, int index) { + setMetadata(parent.getMetadata()); this.primitiveType = primitiveType; this.parent = parent; this.index = index; @@ -112,8 +128,11 @@ public void set(int index, Writable value) { private final ConverterParent parent; private final int index; private final List list = new ArrayList(); + private final Map metadata = new HashMap(); + public RepeatedGroupConverter(GroupType groupType, ConverterParent parent, int index) { + setMetadata(parent.getMetadata()); this.groupType = groupType; this.parent = parent; this.index = index; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java index 3f8e4d7..47cd682 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/DataWritableReadSupport.java @@ -153,7 +153,6 @@ throw new IllegalStateException("ReadContext not initialized properly. " + "Don't know the Hive Schema."); } - final MessageType tableSchema = MessageTypeParser.parseMessageType(metadata.get(HIVE_SCHEMA_KEY)); - return new DataWritableRecordConverter(readContext.getRequestedSchema(), tableSchema); + return new DataWritableRecordConverter(readContext.getRequestedSchema(), metadata); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java index 4e4d7fd..6dc85fa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java @@ -16,10 +16,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.parquet.ProjectionPusher; @@ -246,6 +248,7 @@ protected ParquetInputSplit getSplit( LOG.warn("Skipping split, could not find row group in: " + (FileSplit) oldSplit); split = null; } else { + populateReadMetadata(readContext.getReadSupportMetadata(), fileMetaData, conf); split = new ParquetInputSplit(finalPath, splitStart, splitLength, @@ -261,4 +264,16 @@ protected ParquetInputSplit getSplit( } return split; } + + /** + * Method populates the read metadata, using filemetadata and Hive configuration. + * @param metadata read metadata to populate + * @param fileMetaData parquet file metadata + * @param conf hive configuration + */ + private void populateReadMetadata(Map metadata, FileMetaData fileMetaData, JobConf conf) { + metadata.put("createdBy", fileMetaData.getCreatedBy()); + metadata.put(HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION.varname, + String.valueOf(HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION))); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java index c647b24..9a77dc1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java @@ -28,19 +28,31 @@ static final long SECONDS_PER_MINUTE = 60; static final long MINUTES_PER_HOUR = 60; - private static final ThreadLocal parquetTsCalendar = new ThreadLocal(); + private static final ThreadLocal parquetGMTCalendar = new ThreadLocal(); + private static final ThreadLocal parquetLocalCalendar = new ThreadLocal(); - private static Calendar getCalendar() { + private static Calendar getGMTCalendar() { //Calendar.getInstance calculates the current-time needlessly, so cache an instance. - if (parquetTsCalendar.get() == null) { - parquetTsCalendar.set(Calendar.getInstance(TimeZone.getTimeZone("GMT"))); + if (parquetGMTCalendar.get() == null) { + parquetGMTCalendar.set(Calendar.getInstance(TimeZone.getTimeZone("GMT"))); } - return parquetTsCalendar.get(); + return parquetGMTCalendar.get(); } - public static NanoTime getNanoTime(Timestamp ts) { + private static Calendar getLocalCalendar() { + if (parquetLocalCalendar.get() == null) { + parquetLocalCalendar.set(Calendar.getInstance()); + } + return parquetLocalCalendar.get(); + } + + private static Calendar getCalendar(boolean skipConversion) { + return skipConversion ? getLocalCalendar() : getGMTCalendar(); + } + + public static NanoTime getNanoTime(Timestamp ts, boolean skipConversion) { - Calendar calendar = getCalendar(); + Calendar calendar = getCalendar(skipConversion); calendar.setTime(ts); JDateTime jDateTime = new JDateTime(calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1, //java calendar index starting at 1. @@ -53,15 +65,16 @@ public static NanoTime getNanoTime(Timestamp ts) { long nanos = ts.getNanos(); long nanosOfDay = nanos + NANOS_PER_SECOND * second + NANOS_PER_SECOND * SECONDS_PER_MINUTE * minute + NANOS_PER_SECOND * SECONDS_PER_MINUTE * MINUTES_PER_HOUR * hour; + return new NanoTime(days, nanosOfDay); } - public static Timestamp getTimestamp(NanoTime nt) { + public static Timestamp getTimestamp(NanoTime nt, boolean skipConversion) { int julianDay = nt.getJulianDay(); long nanosOfDay = nt.getTimeOfDayNanos(); JDateTime jDateTime = new JDateTime((double) julianDay); - Calendar calendar = getCalendar(); + Calendar calendar = getCalendar(skipConversion); calendar.set(Calendar.YEAR, jDateTime.getYear()); calendar.set(Calendar.MONTH, jDateTime.getMonth() - 1); //java calender index starting at 1. calendar.set(Calendar.DAY_OF_MONTH, jDateTime.getDay()); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java index 41b5f1c..1d83bf3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java @@ -229,7 +229,7 @@ private void writePrimitive(final Writable value) { recordConsumer.addBinary((Binary.fromByteArray(((BytesWritable) value).getBytes()))); } else if (value instanceof TimestampWritable) { Timestamp ts = ((TimestampWritable) value).getTimestamp(); - NanoTime nt = NanoTimeUtils.getNanoTime(ts); + NanoTime nt = NanoTimeUtils.getNanoTime(ts, false); nt.writeValue(recordConsumer); } else { throw new IllegalArgumentException("Unknown value type: " + value + " " + value.getClass()); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetTimestampUtils.java ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetTimestampUtils.java index 2e788bd..510ffd1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetTimestampUtils.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/parquet/serde/TestParquetTimestampUtils.java @@ -41,10 +41,10 @@ public void testJulianDay() { cal.setTimeZone(TimeZone.getTimeZone("GMT")); Timestamp ts = new Timestamp(cal.getTimeInMillis()); - NanoTime nt = NanoTimeUtils.getNanoTime(ts); + NanoTime nt = NanoTimeUtils.getNanoTime(ts, false); Assert.assertEquals(nt.getJulianDay(), 2440000); - Timestamp tsFetched = NanoTimeUtils.getTimestamp(nt); + Timestamp tsFetched = NanoTimeUtils.getTimestamp(nt, false); Assert.assertEquals(tsFetched, ts); //check if 30 Julian Days between Jan 1, 2005 and Jan 31, 2005. @@ -56,9 +56,9 @@ public void testJulianDay() { cal1.setTimeZone(TimeZone.getTimeZone("GMT")); Timestamp ts1 = new Timestamp(cal1.getTimeInMillis()); - NanoTime nt1 = NanoTimeUtils.getNanoTime(ts1); + NanoTime nt1 = NanoTimeUtils.getNanoTime(ts1, false); - Timestamp ts1Fetched = NanoTimeUtils.getTimestamp(nt1); + Timestamp ts1Fetched = NanoTimeUtils.getTimestamp(nt1, false); Assert.assertEquals(ts1Fetched, ts1); Calendar cal2 = Calendar.getInstance(); @@ -69,9 +69,9 @@ public void testJulianDay() { cal2.setTimeZone(TimeZone.getTimeZone("UTC")); Timestamp ts2 = new Timestamp(cal2.getTimeInMillis()); - NanoTime nt2 = NanoTimeUtils.getNanoTime(ts2); + NanoTime nt2 = NanoTimeUtils.getNanoTime(ts2, false); - Timestamp ts2Fetched = NanoTimeUtils.getTimestamp(nt2); + Timestamp ts2Fetched = NanoTimeUtils.getTimestamp(nt2, false); Assert.assertEquals(ts2Fetched, ts2); Assert.assertEquals(nt2.getJulianDay() - nt1.getJulianDay(), 30); } @@ -90,7 +90,7 @@ public void testNanos() { ts.setNanos(1); //(1*60*60 + 1*60 + 1) * 10e9 + 1 - NanoTime nt = NanoTimeUtils.getNanoTime(ts); + NanoTime nt = NanoTimeUtils.getNanoTime(ts, false); Assert.assertEquals(nt.getTimeOfDayNanos(), 3661000000001L); //case 2: 23:59:59.999999999 @@ -106,7 +106,7 @@ public void testNanos() { ts.setNanos(999999999); //(23*60*60 + 59*60 + 59)*10e9 + 999999999 - nt = NanoTimeUtils.getNanoTime(ts); + nt = NanoTimeUtils.getNanoTime(ts, false); Assert.assertEquals(nt.getTimeOfDayNanos(), 86399999999999L); //case 3: verify the difference. @@ -132,8 +132,8 @@ public void testNanos() { Timestamp ts1 = new Timestamp(cal1.getTimeInMillis()); ts1.setNanos(1); - NanoTime n2 = NanoTimeUtils.getNanoTime(ts2); - NanoTime n1 = NanoTimeUtils.getNanoTime(ts1); + NanoTime n2 = NanoTimeUtils.getNanoTime(ts2, false); + NanoTime n1 = NanoTimeUtils.getNanoTime(ts1, false); Assert.assertEquals(n2.getTimeOfDayNanos() - n1.getTimeOfDayNanos(), 600000000009L); } @@ -157,7 +157,7 @@ public void testTimezone() { * 17:00 PST = 01:00 GMT (if not daylight savings) * (1*60*60 + 1*60 + 1)*10e9 + 1 = 3661000000001 */ - NanoTime nt = NanoTimeUtils.getNanoTime(ts); + NanoTime nt = NanoTimeUtils.getNanoTime(ts, false); long timeOfDayNanos = nt.getTimeOfDayNanos(); Assert.assertTrue(timeOfDayNanos == 61000000001L || timeOfDayNanos == 3661000000001L); @@ -165,39 +165,63 @@ public void testTimezone() { Assert.assertEquals(nt.getJulianDay(), 2440001); } - public void testValues() { + public void testTimezoneValues() { + valueTest(false); + } + + public void testTimezonelessValues() { + valueTest(true); + } + + public void testTimezoneless() { + Timestamp ts1 = Timestamp.valueOf("2011-01-01 00:30:30.111111111"); + NanoTime nt1 = NanoTimeUtils.getNanoTime(ts1, true); + Assert.assertEquals(nt1.getJulianDay(), 2455563); + Assert.assertEquals(nt1.getTimeOfDayNanos(), 1830111111111L); + Timestamp ts1Fetched = NanoTimeUtils.getTimestamp(nt1, true); + Assert.assertEquals(ts1Fetched.toString(), ts1.toString()); + + Timestamp ts2 = Timestamp.valueOf("2011-02-02 08:30:30.222222222"); + NanoTime nt2 = NanoTimeUtils.getNanoTime(ts2, true); + Assert.assertEquals(nt2.getJulianDay(), 2455595); + Assert.assertEquals(nt2.getTimeOfDayNanos(), 30630222222222L); + Timestamp ts2Fetched = NanoTimeUtils.getTimestamp(nt2, true); + Assert.assertEquals(ts2Fetched.toString(), ts2.toString()); + } + + private void valueTest(boolean local) { //exercise a broad range of timestamps close to the present. - verifyTsString("2011-01-01 01:01:01.111111111"); - verifyTsString("2012-02-02 02:02:02.222222222"); - verifyTsString("2013-03-03 03:03:03.333333333"); - verifyTsString("2014-04-04 04:04:04.444444444"); - verifyTsString("2015-05-05 05:05:05.555555555"); - verifyTsString("2016-06-06 06:06:06.666666666"); - verifyTsString("2017-07-07 07:07:07.777777777"); - verifyTsString("2018-08-08 08:08:08.888888888"); - verifyTsString("2019-09-09 09:09:09.999999999"); - verifyTsString("2020-10-10 10:10:10.101010101"); - verifyTsString("2021-11-11 11:11:11.111111111"); - verifyTsString("2022-12-12 12:12:12.121212121"); - verifyTsString("2023-01-02 13:13:13.131313131"); - verifyTsString("2024-02-02 14:14:14.141414141"); - verifyTsString("2025-03-03 15:15:15.151515151"); - verifyTsString("2026-04-04 16:16:16.161616161"); - verifyTsString("2027-05-05 17:17:17.171717171"); - verifyTsString("2028-06-06 18:18:18.181818181"); - verifyTsString("2029-07-07 19:19:19.191919191"); - verifyTsString("2030-08-08 20:20:20.202020202"); - verifyTsString("2031-09-09 21:21:21.212121212"); + verifyTsString("2011-01-01 01:01:01.111111111", local); + verifyTsString("2012-02-02 02:02:02.222222222", local); + verifyTsString("2013-03-03 03:03:03.333333333", local); + verifyTsString("2014-04-04 04:04:04.444444444", local); + verifyTsString("2015-05-05 05:05:05.555555555", local); + verifyTsString("2016-06-06 06:06:06.666666666", local); + verifyTsString("2017-07-07 07:07:07.777777777", local); + verifyTsString("2018-08-08 08:08:08.888888888", local); + verifyTsString("2019-09-09 09:09:09.999999999", local); + verifyTsString("2020-10-10 10:10:10.101010101", local); + verifyTsString("2021-11-11 11:11:11.111111111", local); + verifyTsString("2022-12-12 12:12:12.121212121", local); + verifyTsString("2023-01-02 13:13:13.131313131", local); + verifyTsString("2024-02-02 14:14:14.141414141", local); + verifyTsString("2025-03-03 15:15:15.151515151", local); + verifyTsString("2026-04-04 16:16:16.161616161", local); + verifyTsString("2027-05-05 17:17:17.171717171", local); + verifyTsString("2028-06-06 18:18:18.181818181", local); + verifyTsString("2029-07-07 19:19:19.191919191", local); + verifyTsString("2030-08-08 20:20:20.202020202", local); + verifyTsString("2031-09-09 21:21:21.212121212", local); //test some extreme cases. - verifyTsString("9999-09-09 09:09:09.999999999"); - verifyTsString("0001-01-01 00:00:00.0"); + verifyTsString("9999-09-09 09:09:09.999999999", local); + verifyTsString("0001-01-01 00:00:00.0", local); } - private void verifyTsString(String tsString) { + private void verifyTsString(String tsString, boolean local) { Timestamp ts = Timestamp.valueOf(tsString); - NanoTime nt = NanoTimeUtils.getNanoTime(ts); - Timestamp tsFetched = NanoTimeUtils.getTimestamp(nt); + NanoTime nt = NanoTimeUtils.getNanoTime(ts, local); + Timestamp tsFetched = NanoTimeUtils.getTimestamp(nt, local); Assert.assertEquals(tsString, tsFetched.toString()); } } diff --git ql/src/test/queries/clientpositive/parquet_external_time.q ql/src/test/queries/clientpositive/parquet_external_time.q new file mode 100644 index 0000000..6c9f2d1 --- /dev/null +++ ql/src/test/queries/clientpositive/parquet_external_time.q @@ -0,0 +1,5 @@ +create table timetest_parquet(t timestamp) stored as parquet; + +load data local inpath '../../data/files/parquet_external_time.parq' into table timetest_parquet; + +select * from timetest_parquet; \ No newline at end of file diff --git ql/src/test/results/clientpositive/parquet_external_time.q.out ql/src/test/results/clientpositive/parquet_external_time.q.out new file mode 100644 index 0000000..5f8e6b6 --- /dev/null +++ ql/src/test/results/clientpositive/parquet_external_time.q.out @@ -0,0 +1,25 @@ +PREHOOK: query: create table timetest_parquet(t timestamp) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timetest_parquet +POSTHOOK: query: create table timetest_parquet(t timestamp) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timetest_parquet +PREHOOK: query: load data local inpath '../../data/files/parquet_external_time.parq' into table timetest_parquet +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@timetest_parquet +POSTHOOK: query: load data local inpath '../../data/files/parquet_external_time.parq' into table timetest_parquet +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@timetest_parquet +PREHOOK: query: select * from timetest_parquet +PREHOOK: type: QUERY +PREHOOK: Input: default@timetest_parquet +#### A masked pattern was here #### +POSTHOOK: query: select * from timetest_parquet +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timetest_parquet +#### A masked pattern was here #### +2011-01-01 00:30:30.111111111