diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java index 7a482d968f..6a5a91b55c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java @@ -17,18 +17,10 @@ */ package org.apache.hadoop.hive.ql.optimizer.calcite.translator; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import org.apache.calcite.avatica.util.TimeUnit; import org.apache.calcite.avatica.util.TimeUnitRange; import org.apache.calcite.plan.RelOptCluster; @@ -49,6 +41,7 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.util.ConversionUtil; import org.apache.calcite.util.DateString; import org.apache.calcite.util.NlsString; @@ -104,9 +97,17 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; -import com.google.common.collect.ImmutableMap; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; public class RexNodeConverter { @@ -455,26 +456,50 @@ private RexNode handleExplicitCast(ExprNodeGenericFuncDesc func, List c private List rewriteExtractDateChildren(SqlOperator op, List childRexNodeLst) throws SemanticException { - List newChildRexNodeLst = new ArrayList(); + List newChildRexNodeLst = new ArrayList<>(2); + final boolean isTimestampLevel; if (op == HiveExtractDate.YEAR) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.YEAR)); + isTimestampLevel = false; } else if (op == HiveExtractDate.QUARTER) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.QUARTER)); + isTimestampLevel = false; } else if (op == HiveExtractDate.MONTH) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.MONTH)); + isTimestampLevel = false; } else if (op == HiveExtractDate.WEEK) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.WEEK)); + isTimestampLevel = false; } else if (op == HiveExtractDate.DAY) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.DAY)); + isTimestampLevel = false; } else if (op == HiveExtractDate.HOUR) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.HOUR)); + isTimestampLevel = true; } else if (op == HiveExtractDate.MINUTE) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.MINUTE)); + isTimestampLevel = true; } else if (op == HiveExtractDate.SECOND) { newChildRexNodeLst.add(cluster.getRexBuilder().makeFlag(TimeUnitRange.SECOND)); + isTimestampLevel = true; + } else { + isTimestampLevel = false; } - assert childRexNodeLst.size() == 1; - newChildRexNodeLst.add(childRexNodeLst.get(0)); + + final RexNode child = Iterables.getOnlyElement(childRexNodeLst); + if (SqlTypeUtil.isDatetime(child.getType())) { + newChildRexNodeLst.add(child); + } else { + // We need to add a cast to DATETIME Family + if (isTimestampLevel) { + newChildRexNodeLst.add( + cluster.getRexBuilder().makeCast(cluster.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP), child)); + } else { + newChildRexNodeLst.add( + cluster.getRexBuilder().makeCast(cluster.getTypeFactory().createSqlType(SqlTypeName.DATE), child)); + } + } + return newChildRexNodeLst; } diff --git ql/src/test/queries/clientpositive/druidmini_extractTime.q ql/src/test/queries/clientpositive/druidmini_extractTime.q index 2f7129edeb..e5081851a8 100644 --- ql/src/test/queries/clientpositive/druidmini_extractTime.q +++ ql/src/test/queries/clientpositive/druidmini_extractTime.q @@ -1,3 +1,5 @@ +--! qt:dataset:alltypesorc + SET hive.vectorized.execution.enabled=false; CREATE TABLE druid_table STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' @@ -160,5 +162,27 @@ AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; SELECT EXTRACT(YEAR from `__time`), SUBSTRING(CAST(CAST(`__time` AS DATE) AS STRING), 1, 4) as year_str FROM druid_table WHERE EXTRACT(YEAR from `__time`) >= 1969 AND CAST(EXTRACT(YEAR from `__time`) as STRING) = '1969' LIMIT 1; +-- Test Extract from non datetime column + +create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double); +insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0); +CREATE TABLE druid_test_extract_from_string_table +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "DAY") +AS select +cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` +FROM test_extract_from_string_base_table; + +explain select +year(date_c), month(date_c),day(date_c), +year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table; + +select year(date_c), month(date_c), day(date_c), +year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table; + +DROP TABLE druid_test_extract_from_string_table; +DROP TABLE test_extract_from_string_base_table; DROP TABLE druid_table; \ No newline at end of file diff --git ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out index cf8161f4cb..7bba3840ab 100644 --- ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_extractTime.q.out @@ -1015,6 +1015,109 @@ POSTHOOK: Input: default@druid_table POSTHOOK: Output: hdfs://### HDFS PATH ### 1969 1969 1969 1969 +PREHOOK: query: create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_extract_from_string_base_table +POSTHOOK: query: create table test_extract_from_string_base_table(`timecolumn` timestamp, `date_c` string, `timestamp_c` string, `metric_c` double) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_extract_from_string_base_table +PREHOOK: query: insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test_extract_from_string_base_table +POSTHOOK: query: insert into test_extract_from_string_base_table values ('2015-03-08 00:00:00', '2015-03-10', '2015-03-08 05:30:20', 5.0) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test_extract_from_string_base_table +POSTHOOK: Lineage: test_extract_from_string_base_table.date_c SCRIPT [] +POSTHOOK: Lineage: test_extract_from_string_base_table.metric_c SCRIPT [] +POSTHOOK: Lineage: test_extract_from_string_base_table.timecolumn SCRIPT [] +POSTHOOK: Lineage: test_extract_from_string_base_table.timestamp_c SCRIPT [] +PREHOOK: query: CREATE TABLE druid_test_extract_from_string_table +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "DAY") +AS select +cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` +FROM test_extract_from_string_base_table +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@test_extract_from_string_base_table +PREHOOK: Output: database:default +PREHOOK: Output: default@druid_test_extract_from_string_table +POSTHOOK: query: CREATE TABLE druid_test_extract_from_string_table +STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler' +TBLPROPERTIES ("druid.segment.granularity" = "DAY") +AS select +cast(`timecolumn` as timestamp with local time zone) as `__time`, `date_c`, `timestamp_c`, `metric_c` +FROM test_extract_from_string_base_table +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@test_extract_from_string_base_table +POSTHOOK: Output: database:default +POSTHOOK: Output: default@druid_test_extract_from_string_table +POSTHOOK: Lineage: druid_test_extract_from_string_table.__time EXPRESSION [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:timecolumn, type:timestamp, comment:null), ] +POSTHOOK: Lineage: druid_test_extract_from_string_table.date_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:date_c, type:string, comment:null), ] +POSTHOOK: Lineage: druid_test_extract_from_string_table.metric_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:metric_c, type:double, comment:null), ] +POSTHOOK: Lineage: druid_test_extract_from_string_table.timestamp_c SIMPLE [(test_extract_from_string_base_table)test_extract_from_string_base_table.FieldSchema(name:timestamp_c, type:string, comment:null), ] +PREHOOK: query: explain select +year(date_c), month(date_c),day(date_c), +year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table +PREHOOK: type: QUERY +POSTHOOK: query: explain select +year(date_c), month(date_c),day(date_c), +year(timestamp_c), month(timestamp_c),day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: druid_test_extract_from_string_table + properties: + druid.fieldNames vc,vc0,vc1,vc2,vc3,vc4,vc5,vc6,vc7 + druid.fieldTypes int,int,int,int,int,int,int,int,int + druid.query.json {"queryType":"scan","dataSource":"default.druid_test_extract_from_string_table","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"virtualColumns":[{"type":"expression","name":"vc","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','US/Pacific'),'P1D','','US/Pacific'),'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc0","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','US/Pacific'),'P1D','','US/Pacific'),'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc1","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"date_c\",'','US/Pacific'),'P1D','','US/Pacific'),'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc2","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'P1D','','US/Pacific'),'YEAR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc3","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'P1D','','US/Pacific'),'MONTH','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc4","expression":"timestamp_extract(timestamp_floor(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'P1D','','US/Pacific'),'DAY','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc5","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'HOUR','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc6","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'MINUTE','US/Pacific')","outputType":"LONG"},{"type":"expression","name":"vc7","expression":"timestamp_extract(timestamp_parse(\"timestamp_c\",'','US/Pacific'),'SECOND','US/Pacific')","outputType":"LONG"}],"columns":["vc","vc0","vc1","vc2","vc3","vc4","vc5","vc6","vc7"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: vc (type: int), vc0 (type: int), vc1 (type: int), vc2 (type: int), vc3 (type: int), vc4 (type: int), vc5 (type: int), vc6 (type: int), vc7 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 + ListSink + +PREHOOK: query: select year(date_c), month(date_c), day(date_c), +year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table +PREHOOK: type: QUERY +PREHOOK: Input: default@druid_test_extract_from_string_table +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select year(date_c), month(date_c), day(date_c), +year(timestamp_c), month(timestamp_c), day(timestamp_c), hour(timestamp_c), minute (timestamp_c), second (timestamp_c) +from druid_test_extract_from_string_table +POSTHOOK: type: QUERY +POSTHOOK: Input: default@druid_test_extract_from_string_table +POSTHOOK: Output: hdfs://### HDFS PATH ### +2015 3 10 2015 3 8 5 30 20 +PREHOOK: query: DROP TABLE druid_test_extract_from_string_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@druid_test_extract_from_string_table +PREHOOK: Output: default@druid_test_extract_from_string_table +POSTHOOK: query: DROP TABLE druid_test_extract_from_string_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@druid_test_extract_from_string_table +POSTHOOK: Output: default@druid_test_extract_from_string_table +PREHOOK: query: DROP TABLE test_extract_from_string_base_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test_extract_from_string_base_table +PREHOOK: Output: default@test_extract_from_string_base_table +POSTHOOK: query: DROP TABLE test_extract_from_string_base_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test_extract_from_string_base_table +POSTHOOK: Output: default@test_extract_from_string_base_table PREHOOK: query: DROP TABLE druid_table PREHOOK: type: DROPTABLE PREHOOK: Input: default@druid_table