From e696d9bbed21abca1969065746942d64f400a499 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Mon, 29 Jun 2020 04:41:57 +0530 Subject: [PATCH] [HIVE-23770] Fix Druid filter translation for inverted between --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 6 ++- .../clientpositive/druid_timestamptz2.q | 3 ++ .../druid/druid_timestamptz2.q.out | 39 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 335e25644a..c62415c97e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -201,6 +201,8 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateReduceRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveAggregateSplitRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveCardinalityPreservingJoinRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidPullInvertFromBetweenRule; +import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidPushInvertIntoBetweenRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveDruidRules; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExceptRewriteRule; import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveExpandDistinctAggregatesRule; @@ -2437,6 +2439,7 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv // 7. Apply Druid transformation rules generatePartialProgram(program, false, HepMatchOrder.DEPTH_FIRST, + HiveDruidPullInvertFromBetweenRule.INSTANCE, HiveDruidRules.FILTER_DATE_RANGE_RULE, HiveDruidRules.FILTER, HiveDruidRules.PROJECT_FILTER_TRANSPOSE, HiveDruidRules.AGGREGATE_FILTER_TRANSPOSE, @@ -2450,7 +2453,8 @@ private RelNode applyPostJoinOrderingTransform(RelNode basePlan, RelMetadataProv HiveDruidRules.HAVING_FILTER_RULE, HiveDruidRules.SORT_PROJECT_TRANSPOSE, HiveDruidRules.SORT, - HiveDruidRules.PROJECT_SORT_TRANSPOSE); + HiveDruidRules.PROJECT_SORT_TRANSPOSE, + HiveDruidPushInvertIntoBetweenRule.INSTANCE); // 8. Apply JDBC transformation rules if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) { diff --git a/ql/src/test/queries/clientpositive/druid_timestamptz2.q b/ql/src/test/queries/clientpositive/druid_timestamptz2.q index 29cc02f8c3..effe30fd82 100644 --- a/ql/src/test/queries/clientpositive/druid_timestamptz2.q +++ b/ql/src/test/queries/clientpositive/druid_timestamptz2.q @@ -61,3 +61,6 @@ insert into druid_test_table_utc2 values (cast('2015-03-10 23:59:59' as timestamp with local time zone), 'i3-end', 2); select * FROM druid_test_table_utc2; + +EXPLAIN select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))); +select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))); \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out index d9690319ae..ac017ab966 100644 --- a/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out +++ b/ql/src/test/results/clientpositive/druid/druid_timestamptz2.q.out @@ -207,3 +207,42 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 2015-03-09 23:59:59.0 UTC i2-end 1.0 2015-03-10 00:00:00.0 UTC i3-start 2.0 2015-03-10 23:59:59.0 UTC i3-end 2.0 +PREHOOK: query: EXPLAIN select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))) +PREHOOK: type: QUERY +PREHOOK: Input: druid_test_dst@druid_test_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: EXPLAIN select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))) +POSTHOOK: type: QUERY +POSTHOOK: Input: druid_test_dst@druid_test_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: druid_test_table_1 + properties: + druid.fieldNames interval_marker + druid.fieldTypes string + druid.query.json {"queryType":"scan","dataSource":"druid_test_dst.druid_test_table_1","intervals":["1900-01-01T00:00:00.000Z/3000-01-01T00:00:00.000Z"],"filter":{"type":"not","field":{"type":"bound","dimension":"interval_marker","lower":"i2-start","lowerStrict":false,"upper":"i3-start","upperStrict":false,"ordering":"lexicographic"}},"columns":["interval_marker"],"resultFormat":"compactedList"} + druid.query.type scan + Select Operator + expressions: interval_marker (type: string) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))) +PREHOOK: type: QUERY +PREHOOK: Input: druid_test_dst@druid_test_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select `interval_marker` from druid_test_table_1 WHERE (NOT(((`interval_marker` >= 'i2-start') AND (`interval_marker` <= 'i3-start')))) +POSTHOOK: type: QUERY +POSTHOOK: Input: druid_test_dst@druid_test_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +i1-start +i1-end +i2-end -- 2.20.1 (Apple Git-117)