diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 65264f323f..cdec5ce12d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1682,6 +1682,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_MATERIALIZED_VIEW_SERDE("hive.materializedview.serde", "org.apache.hadoop.hive.ql.io.orc.OrcSerde", "Default SerDe used for materialized views"), + HIVE_ENABLE_JDBC_PUSHDOWN("hive.jdbc.pushdown.enable", true, + "Flag to control enabling pushdown of operators into JDBC connection and subsequent SQL generation\n" + + "using Calcite."), + // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100, ""), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 6e9c68b86e..b3c6806217 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1924,16 +1924,20 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu ); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules"); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, - HepMatchOrder.TOP_DOWN, - JDBCExtractJoinFilterRule.INSTANCE, - JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN, - JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER, - JDBCFilterJoinRule.INSTANCE, - JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE, - JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE, - JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE - ); + if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) { + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, + HepMatchOrder.TOP_DOWN, + JDBCExtractJoinFilterRule.INSTANCE, + JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN, + JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER, + JDBCFilterJoinRule.INSTANCE, + JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE, + JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE, + JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE + ); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: JDBC transformation rules"); + } // 11. Run rules to aid in translation from Calcite tree to Hive tree if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) { diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table2.q b/ql/src/test/queries/clientpositive/external_jdbc_table2.q index 558adf02d8..5059b44c52 100644 --- a/ql/src/test/queries/clientpositive/external_jdbc_table2.q +++ b/ql/src/test/queries/clientpositive/external_jdbc_table2.q @@ -127,6 +127,13 @@ SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2; SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2; +set hive.jdbc.pushdown.enable=false; + +EXPLAIN +SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2; + +SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2; + DROP TABLE db1_ext_auth1; DROP TABLE db2_ext_auth2; DROP TABLE db1_ext_auth2; diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out index 9bfee47b89..13c049fc75 100644 --- a/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table2.q.out @@ -535,6 +535,94 @@ FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2") AS "t" outputColumnNames: _col0, _col1, _col2, _col3 ListSink +PREHOOK: query: SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +20 20 20.0 20.0 +-20 -20 -20.0 -20.0 +100 -15 65.0 -74.0 +44 53 -455.454 330.76 +20 20 20.0 20.0 +-20 8 9.0 11.0 +101 -16 66.0 -75.0 +40 50 -455.4543 330.767 +PREHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: db1_ext_auth1 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: db1_ext_auth2 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 48 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + PREHOOK: query: SELECT * FROM db1_ext_auth1 UNION ALL SELECT * FROM db1_ext_auth2 PREHOOK: type: QUERY PREHOOK: Input: default@db1_ext_auth1