diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 917e565f28b2c9aaea18033ea3b6b20fa41fcd0a..1f55fde18aff8c670a226c2f96605103f6bbbc26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -118,6 +118,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetTableUtils; +import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateTask; import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork; import org.apache.hadoop.hive.ql.lockmgr.DbLockManager; @@ -3815,6 +3816,10 @@ private int alterTableOrSinglePartition(AlterTableDesc alterTbl, Table tbl, Part .get(StatsSetupConst.STATS_GENERATED))) { environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); } + if(alterTbl.getProps().containsKey(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY)) { + NanoTimeUtils.validateTimeZone( + alterTbl.getProps().get(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY)); + } if (part != null) { part.getTPartition().getParameters().putAll(alterTbl.getProps()); } else { @@ -4345,11 +4350,13 @@ private int createTable(Hive db, CreateTableDesc crtTbl) throws HiveException { // to UTC by default (only if the table property is not set) if (tbl.getSerializationLib().equals(ParquetHiveSerDe.class.getName())) { SessionState ss = SessionState.get(); - if (ss.getConf().getBoolVar(ConfVars.HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE)) { - String parquetTimezone = tbl.getProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY); - if (parquetTimezone == null || parquetTimezone.isEmpty()) { + String parquetTimezone = tbl.getProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY); + if (parquetTimezone == null || parquetTimezone.isEmpty()) { + if (ss.getConf().getBoolVar(ConfVars.HIVE_PARQUET_INT96_DEFAULT_UTC_WRITE_ZONE)) { tbl.setProperty(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY, ParquetTableUtils.PARQUET_INT96_NO_ADJUSTMENT_ZONE); } + } else { + NanoTimeUtils.validateTimeZone(parquetTimezone); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 004bb2f60299a0635b8f9ca7649ead00b8e16d08..8d99649f99adb3294e87567c058537f1e82b4e68 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveRecordReader; +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetTableUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.parse.SplitSample; @@ -329,6 +330,7 @@ static void setFetchOperatorContext(JobConf conf, List paths) { final FetchInputFormatSplit target = iterSplits.next(); + ParquetTableUtils.ensureTablePropertySet(job, currDesc.getTableDesc()); @SuppressWarnings("unchecked") final RecordReader reader = target.getRecordReader(job); if (hasVC || work.getSplitSample() != null) { @@ -367,6 +369,7 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException Class formatter = currDesc.getInputFileFormatClass(); Utilities.copyTableJobPropertiesToConf(currDesc.getTableDesc(), job); + ParquetTableUtils.ensureTablePropertySet(job, currDesc.getTableDesc()); InputFormat inputFormat = getInputFormatFromCache(formatter, job); InputSplit[] splits = inputFormat.getSplits(job, 1); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index 1bd4db7805689ae1f91921ffbb5ff7da59f4bf60..f79a592dcb73d709a9a190fb1d3a6ff061efe45d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -69,7 +69,7 @@ static final String HIVE_DEBUG_RECURSIVE = "HIVE_DEBUG_RECURSIVE"; static final String HIVE_MAIN_CLIENT_DEBUG_OPTS = "HIVE_MAIN_CLIENT_DEBUG_OPTS"; static final String HIVE_CHILD_CLIENT_DEBUG_OPTS = "HIVE_CHILD_CLIENT_DEBUG_OPTS"; - static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive", "hive.query.id"}; + static final String[] HIVE_SYS_PROP = {"build.dir", "build.dir.hive", "hive.query.id", "user.timezone"}; private transient ContentSummary inputSummary = null; private transient boolean runningViaChild = false; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java index f4fadbb61bf45f62945700284c0b050f0984b696..fbed1a6f7b5201ac7f3751a843b128be53dd3c20 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java @@ -14,8 +14,15 @@ package org.apache.hadoop.hive.ql.io.parquet; import java.io.IOException; +import java.util.Map; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; +import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; +import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetTableUtils; +import org.apache.hadoop.hive.ql.plan.MapWork; +import org.apache.hadoop.hive.ql.plan.PartitionDesc; +import org.apache.hadoop.mapred.FileSplit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -61,6 +68,33 @@ protected MapredParquetInputFormat(final ParquetInputFormat input final org.apache.hadoop.mapred.JobConf job, final org.apache.hadoop.mapred.Reporter reporter ) throws IOException { + + PartitionDesc part = null; + Path filePath = ((FileSplit) split).getPath(); + try { + MapWork mapWork = Utilities.getMapWork(job); + if(mapWork != null) { + LOG.debug("Trying to find partition in MapWork for path " + filePath); + Map pathToPartitionInfo = mapWork.getPathToPartitionInfo(); + + part = HiveFileFormatUtils + .getPartitionDescFromPathRecursively(pathToPartitionInfo, filePath, null); + LOG.debug("Partition found " + part); + } + } catch (AssertionError ae) { + LOG.debug("Cannot get partition description from " + filePath + + " because " + ae.getMessage()); + part = null; + } catch (Exception e) { + LOG.debug("Cannot get partition description from " + filePath + + " because " + e.getMessage()); + part = null; + } + + if (part != null && part.getTableDesc() != null) { + ParquetTableUtils.ensureTablePropertySet(job, part.getTableDesc()); + } + try { if (Utilities.getUseVectorizedInputFileFormat(job)) { if (LOG.isDebugEnabled()) { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java index 2954601ce5bb25905cdb29ca0ca4551c2ca12b95..be9fb10893134acaacc2e07d7d230a8bbed4c462 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java @@ -178,8 +178,7 @@ protected void setTimeZoneConversion(Configuration configuration, Path finalPath } else { // TABLE_PARQUET_INT96_TIMEZONE is a table property used to detect what timezone conversion // to use when reading Parquet timestamps. - timeZoneID = configuration.get(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY, - TimeZone.getDefault().getID()); + timeZoneID = configuration.get(ParquetTableUtils.PARQUET_INT96_WRITE_ZONE_PROPERTY); NanoTimeUtils.validateTimeZone(timeZoneID); } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetTableUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetTableUtils.java index b339cc4347eea143dca2f6d98f9aa7777afdc427..09a99913be1955187b1045776a6681818668527a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetTableUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetTableUtils.java @@ -13,6 +13,12 @@ */ package org.apache.hadoop.hive.ql.io.parquet.serde; +import org.apache.hadoop.hive.ql.plan.TableDesc; +import org.apache.hadoop.mapred.JobConf; + +import java.util.Properties; +import java.util.TimeZone; + public class ParquetTableUtils { // Parquet table properties public static final String PARQUET_INT96_WRITE_ZONE_PROPERTY = "parquet.mr.int96.write.zone"; @@ -20,4 +26,21 @@ // This is not a TimeZone we convert into and print out, rather a delta, an adjustment we use. // More precisely the lack of an adjustment in case of UTC public static final String PARQUET_INT96_NO_ADJUSTMENT_ZONE = "UTC"; + + /** + * Propagates the parquet timezone property to the job configuration from the table property + * or sets the default + * @param jc + * @param table + */ + public static void ensureTablePropertySet(JobConf jc, TableDesc table) { + if (table != null && jc != null) { + Properties tableProps = table.getProperties(); + if (tableProps.containsKey(PARQUET_INT96_WRITE_ZONE_PROPERTY)) { + jc.set(PARQUET_INT96_WRITE_ZONE_PROPERTY, tableProps.getProperty(PARQUET_INT96_WRITE_ZONE_PROPERTY)); + } else { + jc.set(PARQUET_INT96_WRITE_ZONE_PROPERTY, TimeZone.getDefault().getID()); + } + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java index dbd6fb3d0bc8c753abf86e99b52377617f248b5a..9ec7da02cf591dfd40306199f399f93093e15334 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/timestamp/NanoTimeUtils.java @@ -165,6 +165,9 @@ public static Timestamp getTimestamp(NanoTime nt, Calendar calendar) { * @param timeZoneID */ public static void validateTimeZone(String timeZoneID) { + if(timeZoneID == null) { + throw new IllegalStateException("Missing timezone id for parquet int96 conversion!"); + } if (TimeZone.getTimeZone(timeZoneID).getID().equals("GMT") && !"GMT".equals(timeZoneID)) { throw new IllegalStateException( diff --git ql/src/test/queries/clientnegative/parquet_int96_alter_invalid_timezone.q ql/src/test/queries/clientnegative/parquet_int96_alter_invalid_timezone.q new file mode 100644 index 0000000000000000000000000000000000000000..2de92ad41cf76d3c2372f6741d44b7d99d09bb76 --- /dev/null +++ ql/src/test/queries/clientnegative/parquet_int96_alter_invalid_timezone.q @@ -0,0 +1,5 @@ +-- alter table to invalid table property +create table timestamps (ts timestamp) stored as parquet; +alter table timestamps set tblproperties ('parquet.mr.int96.write.zone'='Invalid'); + +drop table timestamps; diff --git ql/src/test/queries/clientnegative/parquet_int96_create_invalid_timezone.q ql/src/test/queries/clientnegative/parquet_int96_create_invalid_timezone.q new file mode 100644 index 0000000000000000000000000000000000000000..ffba084f46c0bf6aba1000bac629b5abc82fa1b2 --- /dev/null +++ ql/src/test/queries/clientnegative/parquet_int96_create_invalid_timezone.q @@ -0,0 +1,3 @@ +-- create table with invalid table property +create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='Invalid'); + diff --git ql/src/test/queries/clientpositive/parquet_int96_timestamp.q ql/src/test/queries/clientpositive/parquet_int96_timestamp.q index 6eadd1b0a3313cbba7a798890b802baae302749e..d0640fa72c2b2525c33a5b81256f97b555b156b3 100644 --- ql/src/test/queries/clientpositive/parquet_int96_timestamp.q +++ ql/src/test/queries/clientpositive/parquet_int96_timestamp.q @@ -37,10 +37,31 @@ drop table timestamps; -- read/write timestamps with timezone specified in table properties create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST'); insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1; +insert into table timestamps values('2017-01-01 01:01:01'); +-- parquet timezone flag set in the fetch operator select * from timestamps; +-- parquet timezone flag set in MapredParquetInputFormat +select * from timestamps order by ts; +select * from timestamps where ts = cast('2016-01-01 01:01:01' as timestamp); +-- using udfs +select year(ts), day(ts), hour(ts), ts from timestamps; describe formatted timestamps; drop table timestamps; +-- read timestamps with different timezones specified in two table properties +create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST'); +insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1; +insert into table timestamps values('2017-01-01 01:01:01'); +create table timestamps2 (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+2'); +insert into table timestamps2 select cast('2016-01-01 01:01:01' as timestamp) limit 1; +insert into table timestamps2 values('2017-01-01 01:01:01'); +-- parquet timezone flag set in the MapredLocalTask +select * from timestamps a inner join timestamps2 b on a.ts = b.ts; +describe formatted timestamps; +drop table timestamps; +describe formatted timestamps2; +drop table timestamps2; + -- read timestamps written by Impala create table timestamps (ts timestamp) stored as parquet; load data local inpath '../../data/files/impala_int96_timestamp.parq' overwrite into table timestamps; diff --git ql/src/test/results/clientnegative/parquet_int96_alter_invalid_timezone.q.out ql/src/test/results/clientnegative/parquet_int96_alter_invalid_timezone.q.out new file mode 100644 index 0000000000000000000000000000000000000000..97d61a2a9a4e5687738df515d0dac24a01f473b1 --- /dev/null +++ ql/src/test/results/clientnegative/parquet_int96_alter_invalid_timezone.q.out @@ -0,0 +1,13 @@ +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: alter table timestamps set tblproperties ('parquet.mr.int96.write.zone'='Invalid') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unexpected timezone id found for parquet int96 conversion: Invalid diff --git ql/src/test/results/clientnegative/parquet_int96_create_invalid_timezone.q.out ql/src/test/results/clientnegative/parquet_int96_create_invalid_timezone.q.out new file mode 100644 index 0000000000000000000000000000000000000000..d619ce62ca8edeee92293d3dfb0829fdd5934288 --- /dev/null +++ ql/src/test/results/clientnegative/parquet_int96_create_invalid_timezone.q.out @@ -0,0 +1,5 @@ +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='Invalid') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unexpected timezone id found for parquet int96 conversion: Invalid diff --git ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out index b9a3664458a83f1856e4bc59eba5d56665df61cc..83f1a692b5c0e5f1f8323f6b7fd8b89fd8dab9a3 100644 --- ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out +++ ql/src/test/results/clientpositive/parquet_int96_timestamp.q.out @@ -305,6 +305,13 @@ POSTHOOK: type: QUERY POSTHOOK: Input: _dummy_database@_dummy_table POSTHOOK: Output: default@timestamps POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] PREHOOK: query: select * from timestamps PREHOOK: type: QUERY PREHOOK: Input: default@timestamps @@ -314,6 +321,36 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@timestamps #### A masked pattern was here #### 2016-01-01 01:01:01 +2017-01-01 01:01:01 +PREHOOK: query: select * from timestamps order by ts +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps order by ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +2017-01-01 01:01:01 +PREHOOK: query: select * from timestamps where ts = cast('2016-01-01 01:01:01' as timestamp) +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps where ts = cast('2016-01-01 01:01:01' as timestamp) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: select year(ts), day(ts), hour(ts), ts from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select year(ts), day(ts), hour(ts), ts from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016 1 1 2016-01-01 01:01:01 +2017 1 1 2017-01-01 01:01:01 PREHOOK: query: describe formatted timestamps PREHOOK: type: DESCTABLE PREHOOK: Input: default@timestamps @@ -332,11 +369,114 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 1 - numRows 1 + numFiles 2 + numRows 2 parquet.mr.int96.write.zone PST - rawDataSize 1 - totalSize 272 + rawDataSize 2 + totalSize 544 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: create table timestamps2 (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: create table timestamps2 (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps2 +PREHOOK: query: insert into table timestamps2 select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: insert into table timestamps2 select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps2 +POSTHOOK: Lineage: timestamps2.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps2 values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: insert into table timestamps2 values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps2 +POSTHOOK: Lineage: timestamps2.ts EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from timestamps a inner join timestamps2 b on a.ts = b.ts +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +PREHOOK: Input: default@timestamps2 +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps a inner join timestamps2 b on a.ts = b.ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +POSTHOOK: Input: default@timestamps2 +#### A masked pattern was here #### +2016-01-01 01:01:01 2016-01-01 01:01:01 +2017-01-01 01:01:01 2017-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 2 + parquet.mr.int96.write.zone PST + rawDataSize 2 + totalSize 544 #### A masked pattern was here #### # Storage Information @@ -357,6 +497,49 @@ POSTHOOK: query: drop table timestamps POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@timestamps POSTHOOK: Output: default@timestamps +PREHOOK: query: describe formatted timestamps2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps2 +POSTHOOK: query: describe formatted timestamps2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps2 +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 2 + parquet.mr.int96.write.zone GMT+2 + rawDataSize 2 + totalSize 544 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps2 +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: drop table timestamps2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps2 +POSTHOOK: Output: default@timestamps2 PREHOOK: query: create table timestamps (ts timestamp) stored as parquet PREHOOK: type: CREATETABLE PREHOOK: Output: database:default diff --git ql/src/test/results/clientpositive/spark/parquet_int96_timestamp.q.out ql/src/test/results/clientpositive/spark/parquet_int96_timestamp.q.out new file mode 100644 index 0000000000000000000000000000000000000000..83f1a692b5c0e5f1f8323f6b7fd8b89fd8dab9a3 --- /dev/null +++ ql/src/test/results/clientpositive/spark/parquet_int96_timestamp.q.out @@ -0,0 +1,718 @@ +PREHOOK: query: create table dummy (id int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dummy +POSTHOOK: query: create table dummy (id int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@dummy +PREHOOK: query: insert into table dummy values (1) +PREHOOK: type: QUERY +PREHOOK: Output: default@dummy +POSTHOOK: query: insert into table dummy values (1) +POSTHOOK: type: QUERY +POSTHOOK: Output: default@dummy +POSTHOOK: Lineage: dummy.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + parquet.mr.int96.write.zone UTC + rawDataSize 1 + totalSize 272 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + parquet.mr.int96.write.zone PST + rawDataSize 1 + totalSize 272 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + rawDataSize 1 + totalSize 272 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='CST') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='CST') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 1 + numRows 1 + parquet.mr.int96.write.zone CST + rawDataSize 1 + totalSize 272 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +2017-01-01 01:01:01 +PREHOOK: query: select * from timestamps order by ts +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps order by ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +2017-01-01 01:01:01 +PREHOOK: query: select * from timestamps where ts = cast('2016-01-01 01:01:01' as timestamp) +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps where ts = cast('2016-01-01 01:01:01' as timestamp) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: select year(ts), day(ts), hour(ts), ts from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select year(ts), day(ts), hour(ts), ts from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016 1 1 2016-01-01 01:01:01 +2017 1 1 2017-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 2 + parquet.mr.int96.write.zone PST + rawDataSize 2 + totalSize 544 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='PST') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps +POSTHOOK: query: insert into table timestamps values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps +POSTHOOK: Lineage: timestamps.ts EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: create table timestamps2 (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: create table timestamps2 (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps2 +PREHOOK: query: insert into table timestamps2 select cast('2016-01-01 01:01:01' as timestamp) limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: insert into table timestamps2 select cast('2016-01-01 01:01:01' as timestamp) limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@timestamps2 +POSTHOOK: Lineage: timestamps2.ts EXPRESSION [] +PREHOOK: query: insert into table timestamps2 values('2017-01-01 01:01:01') +PREHOOK: type: QUERY +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: insert into table timestamps2 values('2017-01-01 01:01:01') +POSTHOOK: type: QUERY +POSTHOOK: Output: default@timestamps2 +POSTHOOK: Lineage: timestamps2.ts EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: select * from timestamps a inner join timestamps2 b on a.ts = b.ts +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +PREHOOK: Input: default@timestamps2 +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps a inner join timestamps2 b on a.ts = b.ts +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +POSTHOOK: Input: default@timestamps2 +#### A masked pattern was here #### +2016-01-01 01:01:01 2016-01-01 01:01:01 +2017-01-01 01:01:01 2017-01-01 01:01:01 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 2 + parquet.mr.int96.write.zone PST + rawDataSize 2 + totalSize 544 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: describe formatted timestamps2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps2 +POSTHOOK: query: describe formatted timestamps2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps2 +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 2 + numRows 2 + parquet.mr.int96.write.zone GMT+2 + rawDataSize 2 + totalSize 544 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps2 +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: drop table timestamps2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps2 +POSTHOOK: Output: default@timestamps2 +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: load data local inpath '../../data/files/impala_int96_timestamp.parq' overwrite into table timestamps +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@timestamps +POSTHOOK: query: load data local inpath '../../data/files/impala_int96_timestamp.parq' overwrite into table timestamps +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@timestamps +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+10') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+10') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: load data local inpath '../../data/files/impala_int96_timestamp.parq' overwrite into table timestamps +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@timestamps +POSTHOOK: query: load data local inpath '../../data/files/impala_int96_timestamp.parq' overwrite into table timestamps +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@timestamps +PREHOOK: query: select * from timestamps +PREHOOK: type: QUERY +PREHOOK: Input: default@timestamps +#### A masked pattern was here #### +POSTHOOK: query: select * from timestamps +POSTHOOK: type: QUERY +POSTHOOK: Input: default@timestamps +#### A masked pattern was here #### +2016-01-01 01:01:01 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+10') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps +POSTHOOK: query: create table timestamps (ts timestamp) stored as parquet tblproperties('parquet.mr.int96.write.zone'='GMT+10') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps +PREHOOK: query: create table timestamps2 like timestamps +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: create table timestamps2 like timestamps +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@timestamps2 +PREHOOK: query: describe formatted timestamps +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps +POSTHOOK: query: describe formatted timestamps +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + parquet.mr.int96.write.zone GMT+10 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: describe formatted timestamps2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@timestamps2 +POSTHOOK: query: describe formatted timestamps2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@timestamps2 +# col_name data_type comment + +ts timestamp + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: MANAGED_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + numFiles 0 + numRows 0 + parquet.mr.int96.write.zone GMT+10 + rawDataSize 0 + totalSize 0 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe +InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: drop table timestamps +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps +PREHOOK: Output: default@timestamps +POSTHOOK: query: drop table timestamps +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps +POSTHOOK: Output: default@timestamps +PREHOOK: query: drop table timestamps2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@timestamps2 +PREHOOK: Output: default@timestamps2 +POSTHOOK: query: drop table timestamps2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@timestamps2 +POSTHOOK: Output: default@timestamps2 +PREHOOK: query: drop table if exists dummy +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@dummy +PREHOOK: Output: default@dummy +POSTHOOK: query: drop table if exists dummy +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@dummy +POSTHOOK: Output: default@dummy