diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java index 4a7952eb4e..5495822118 100644 --- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java +++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java @@ -121,6 +121,9 @@ public void initialize(Configuration configuration, Properties properties) throw final List columnTypes = new ArrayList<>(); List inspectors = new ArrayList<>(); + final TimestampLocalTZTypeInfo tsTZTypeInfo = new TimestampLocalTZTypeInfo( + configuration.get(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE.varname)); + // Druid query String druidQuery = properties.getProperty(Constants.DRUID_QUERY_JSON); if (druidQuery == null) { @@ -137,8 +140,9 @@ public void initialize(Configuration configuration, Properties properties) throw "') not specified in create table; list of columns is : " + properties.getProperty(serdeConstants.LIST_COLUMNS)); } - columnTypes.addAll(Lists.transform(Utilities.getColumnTypes(properties), - type -> TypeInfoFactory.getPrimitiveTypeInfo(type) + columnTypes.addAll(Lists.transform( + Lists.transform(Utilities.getColumnTypes(properties), type -> TypeInfoFactory.getPrimitiveTypeInfo(type)), + e -> e instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : e )); inspectors.addAll(Lists.transform(columnTypes, (Function) type -> PrimitiveObjectInspectorFactory @@ -179,7 +183,7 @@ public void initialize(Configuration configuration, Properties properties) throw if (columnInfo.getKey().equals(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN)) { // Special handling for timestamp column columnNames.add(columnInfo.getKey()); // field name - PrimitiveTypeInfo type = TypeInfoFactory.timestampLocalTZTypeInfo; // field type + PrimitiveTypeInfo type = tsTZTypeInfo; // field type columnTypes.add(type); inspectors .add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type)); @@ -188,7 +192,7 @@ public void initialize(Configuration configuration, Properties properties) throw columnNames.add(columnInfo.getKey()); // field name PrimitiveTypeInfo type = DruidSerDeUtils.convertDruidToHiveType( columnInfo.getValue().getType()); // field type - columnTypes.add(type); + columnTypes.add(type instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : type); inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type)); } columns = columnNames.toArray(new String[columnNames.size()]); @@ -211,19 +215,21 @@ public void initialize(Configuration configuration, Properties properties) throw List propColumnNames = Utilities.getColumnNames(properties); List propColumnTypes = Utilities.getColumnTypes(properties); for (int i = 0; i < propColumnNames.size(); i++) { - mapColumnNamesTypes.put( - propColumnNames.get(i), - TypeInfoFactory.getPrimitiveTypeInfo(propColumnTypes.get(i))); + PrimitiveTypeInfo type = TypeInfoFactory.getPrimitiveTypeInfo(propColumnTypes.get(i)); + if (type instanceof TimestampLocalTZTypeInfo) { + type = tsTZTypeInfo; + } + mapColumnNamesTypes.put(propColumnNames.get(i), type); } } switch (query.getType()) { case Query.TIMESERIES: - inferSchema((TimeseriesQuery) query, columnNames, columnTypes, + inferSchema((TimeseriesQuery) query, tsTZTypeInfo, columnNames, columnTypes, mapColumnNamesTypes.build()); break; case Query.TOPN: - inferSchema((TopNQuery) query, columnNames, columnTypes, + inferSchema((TopNQuery) query, tsTZTypeInfo, columnNames, columnTypes, mapColumnNamesTypes.build()); break; case Query.SELECT: @@ -232,11 +238,11 @@ public void initialize(Configuration configuration, Properties properties) throw if (org.apache.commons.lang3.StringUtils.isEmpty(address)) { throw new SerDeException("Druid broker address not specified in configuration"); } - inferSchema((SelectQuery) query, columnNames, columnTypes, address, + inferSchema((SelectQuery) query, tsTZTypeInfo, columnNames, columnTypes, address, mapColumnNamesTypes.build()); break; case Query.GROUP_BY: - inferSchema((GroupByQuery) query, columnNames, columnTypes, + inferSchema((GroupByQuery) query, tsTZTypeInfo, columnNames, columnTypes, mapColumnNamesTypes.build()); break; default: @@ -299,12 +305,12 @@ protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQ } /* Timeseries query */ - private void inferSchema(TimeseriesQuery query, + private void inferSchema(TimeseriesQuery query, TimestampLocalTZTypeInfo timeColumnTypeInfo, List columnNames, List columnTypes, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); + columnTypes.add(timeColumnTypeInfo); // Aggregator columns for (AggregatorFactory af : query.getAggregatorSpecs()) { columnNames.add(af.getName()); @@ -327,12 +333,12 @@ private void inferSchema(TimeseriesQuery query, } /* TopN query */ - private void inferSchema(TopNQuery query, + private void inferSchema(TopNQuery query, TimestampLocalTZTypeInfo timeColumnTypeInfo, List columnNames, List columnTypes, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); + columnTypes.add(timeColumnTypeInfo); // Dimension column columnNames.add(query.getDimensionSpec().getOutputName()); columnTypes.add(TypeInfoFactory.stringTypeInfo); @@ -358,13 +364,13 @@ private void inferSchema(TopNQuery query, } /* Select query */ - private void inferSchema(SelectQuery query, + private void inferSchema(SelectQuery query, TimestampLocalTZTypeInfo timeColumnTypeInfo, List columnNames, List columnTypes, String address, Map mapColumnNamesTypes) throws SerDeException { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); + columnTypes.add(timeColumnTypeInfo); // Dimension columns for (DimensionSpec ds : query.getDimensions()) { columnNames.add(ds.getOutputName()); @@ -401,12 +407,12 @@ private void inferSchema(SelectQuery query, } /* GroupBy query */ - private void inferSchema(GroupByQuery query, + private void inferSchema(GroupByQuery query, TimestampLocalTZTypeInfo timeColumnTypeInfo, List columnNames, List columnTypes, Map mapColumnNamesTypes) { // Timestamp column columnNames.add(DruidStorageHandlerUtils.DEFAULT_TIMESTAMP_COLUMN); - columnTypes.add(TypeInfoFactory.timestampLocalTZTypeInfo); + columnTypes.add(timeColumnTypeInfo); // Dimension columns for (DimensionSpec ds : query.getDimensions()) { columnNames.add(ds.getOutputName()); diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index bd2416f4eb..25636c819f 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -1600,5 +1600,6 @@ spark.perf.disabled.query.files=query14.q,\ druid.query.files=druidmini_test1.q,\ druidmini_test_insert.q,\ - druidmini_mv.q + druidmini_mv.q,\ + druid_timestamptz.q diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz.q b/ql/src/test/queries/clientpositive/druid_timestamptz.q new file mode 100644 index 0000000000..1e963f98d4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/druid_timestamptz.q @@ -0,0 +1,20 @@ +set hive.fetch.task.conversion=more; + +drop table tstz1; + +create table tstz1(`__time` timestamp with local time zone, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR"); + +insert into table tstz1 +values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone), 'Bill', 10); + +select `__time` from tstz1; +select cast(`__time` as timestamp) from tstz1; +select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); + +set time zone UTC; + +select `__time` from tstz1; +select cast(`__time` as timestamp) from tstz1; +select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone); diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out new file mode 100644 index 0000000000..0ffaaf495c --- /dev/null +++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz.q.out @@ -0,0 +1,80 @@ +PREHOOK: query: drop table tstz1 +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstz1 +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstz1(`__time` timestamp with local time zone, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@tstz1 +POSTHOOK: query: create table tstz1(`__time` timestamp with local time zone, n string, v integer) +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "HOUR") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@tstz1 +PREHOOK: query: insert into table tstz1 +values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone), 'Bill', 10) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@tstz1 +POSTHOOK: query: insert into table tstz1 +values(cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone), 'Bill', 10) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@tstz1 +PREHOOK: query: select `__time` from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select `__time` from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34.0 US/Pacific +PREHOOK: query: select cast(`__time` as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(`__time` as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34 +PREHOOK: query: select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 12:26:34 +PREHOOK: query: select `__time` from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select `__time` from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 20:26:34.0 UTC +PREHOOK: query: select cast(`__time` as timestamp) from tstz1 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(`__time` as timestamp) from tstz1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 20:26:34 +PREHOOK: query: select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +PREHOOK: type: QUERY +PREHOOK: Input: default@tstz1 +#### A masked pattern was here #### +POSTHOOK: query: select cast(`__time` as timestamp) from tstz1 where `__time` >= cast('2016-01-03 12:26:34 America/Los_Angeles' as timestamp with local time zone) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstz1 +#### A masked pattern was here #### +2016-01-03 20:26:34