diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 0dea0996c9..1dc14cf302 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1702,7 +1702,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_ENABLE_JDBC_PUSHDOWN("hive.jdbc.pushdown.enable", true, "Flag to control enabling pushdown of operators into JDBC connection and subsequent SQL generation\n" + - "using Calcite."), + "using Calcite"), + HIVE_ENABLE_JDBC_NON_PARALLELIZABLE_PUSHDOWN("hive.jdbc.pushdown.complex.enable", true, + "Flag to control enabling pushdown of operators using Calcite that prevent splitting results\n" + + "retrieval in the JDBC storage handler"), // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row, // need to remove by hive .13. Also, do not change default (see SMB operator) diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index a237745487..90316629c2 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -525,6 +525,7 @@ minillaplocal.query.files=\ external_jdbc_auth.q,\ external_jdbc_table.q,\ external_jdbc_table2.q,\ + external_jdbc_table3.q,\ external_jdbc_table_partition.q,\ external_jdbc_table_typeconversion.q,\ fullouter_mapjoin_1_optimized.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index de88783584..db5a7ddb7e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1910,11 +1910,10 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu } // 8. convert SemiJoin + GBy to SemiJoin - perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); - calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, - HiveRemoveGBYSemiJoinRule.INSTANCE); - perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Removal of gby from semijoin"); - + perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); + calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, false, mdProvider.getMetadataProvider(), null, + HiveRemoveGBYSemiJoinRule.INSTANCE); + perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Removal of gby from semijoin"); // 9. Run rule to fix windowing issue when it is done over // aggregation columns (HIVE-10627) @@ -1947,17 +1946,22 @@ public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlu perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: Druid transformation rules"); if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_PUSHDOWN)) { + List rules = Lists.newArrayList(); + rules.add(JDBCExtractJoinFilterRule.INSTANCE); + rules.add(JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN); + rules.add(JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER); + rules.add(JDBCFilterJoinRule.INSTANCE); + rules.add(JDBCFilterPushDownRule.INSTANCE); + rules.add(JDBCProjectPushDownRule.INSTANCE); + if (conf.getBoolVar(ConfVars.HIVE_ENABLE_JDBC_NON_PARALLELIZABLE_PUSHDOWN)) { + rules.add(JDBCJoinPushDownRule.INSTANCE); + rules.add(JDBCUnionPushDownRule.INSTANCE); + rules.add(JDBCAggregationPushDownRule.INSTANCE); + rules.add(JDBCSortPushDownRule.INSTANCE); + } perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER); calciteOptimizedPlan = hepPlan(calciteOptimizedPlan, true, mdProvider.getMetadataProvider(), null, - HepMatchOrder.TOP_DOWN, - JDBCExtractJoinFilterRule.INSTANCE, - JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_JOIN, - JDBCAbstractSplitFilterRule.SPLIT_FILTER_ABOVE_CONVERTER, - JDBCFilterJoinRule.INSTANCE, - JDBCJoinPushDownRule.INSTANCE, JDBCUnionPushDownRule.INSTANCE, - JDBCFilterPushDownRule.INSTANCE, JDBCProjectPushDownRule.INSTANCE, - JDBCAggregationPushDownRule.INSTANCE, JDBCSortPushDownRule.INSTANCE - ); + HepMatchOrder.TOP_DOWN, rules.toArray(new RelOptRule[rules.size()])); perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Calcite: JDBC transformation rules"); } diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table3.q b/ql/src/test/queries/clientpositive/external_jdbc_table3.q new file mode 100644 index 0000000000..398e2d8bf6 --- /dev/null +++ b/ql/src/test/queries/clientpositive/external_jdbc_table3.q @@ -0,0 +1,100 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +set hive.jdbc.pushdown.complex.enable=false; + + +CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'; + +FROM src +SELECT +dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;create=true','user1','passwd1', +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1; + +FROM src +SELECT +dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1; + + + +CREATE EXTERNAL TABLE db1_ext_auth3_1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +); + +CREATE EXTERNAL TABLE db1_ext_auth3_2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +); + + +SELECT * FROM db1_ext_auth3_1; + +SELECT * FROM db1_ext_auth3_2; + +EXPLAIN +SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20; + +SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20; + +EXPLAIN +SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100; + +SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100; + +DROP TABLE db1_ext_auth3_1; +DROP TABLE db1_ext_auth3_2; + +FROM src +SELECT +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth3_1','user1','passwd1', +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1; diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out new file mode 100644 index 0000000000..dd1cda64b7 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table3.q.out @@ -0,0 +1,407 @@ +PREHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: dboutput +POSTHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: dboutput +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@db1_ext_auth3_1 +POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@db1_ext_auth3_1 +PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@db1_ext_auth3_2 +POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth3_2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@db1_ext_auth3_2 +PREHOOK: query: SELECT * FROM db1_ext_auth3_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth3_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_1 +#### A masked pattern was here #### +-20 -20 -20.0 -20.0 +100 -15 65.0 -74.0 +20 20 20.0 20.0 +44 53 -455.454 330.76 +PREHOOK: query: SELECT * FROM db1_ext_auth3_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth3_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +-20 8 9.0 11.0 +101 -16 66.0 -75.0 +20 20 20.0 20.0 +40 50 -455.4543 330.767 +PREHOOK: query: EXPLAIN +SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_1 +PREHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_1 +POSTHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: db1_ext_auth3_1 + properties: + hive.sql.query SELECT "ikey", "fkey" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1" +WHERE "ikey" > 20 + hive.sql.query.fieldNames ikey,fkey + hive.sql.query.fieldTypes int,float + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), fkey (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: float) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: db1_ext_auth3_2 + properties: + hive.sql.query SELECT "ikey", "bkey" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2" +WHERE "ikey" > 20 + hive.sql.query.fieldNames ikey,bkey + hive.sql.query.fieldTypes int,bigint + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), bkey (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col1, _col3 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: float), _col3 (type: bigint) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_1 +PREHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT db1_ext_auth3_1.fkey, db1_ext_auth3_2.bkey FROM db1_ext_auth3_1 JOIN db1_ext_auth3_2 WHERE db1_ext_auth3_1.ikey = db1_ext_auth3_2.ikey AND db1_ext_auth3_1.ikey > 20 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_1 +POSTHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN +SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_1 +PREHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_1 +POSTHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Union 2 (CONTAINS) + Map 3 <- Union 2 (CONTAINS) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: db1_ext_auth3_1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: bkey (type: bigint), fkey (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: db1_ext_auth3_2 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: bkey (type: bigint), fkey (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 24 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized, llap + LLAP IO: no inputs + Union 2 + Vertex: Union 2 + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth3_1 +PREHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT bkey, fkey FROM db1_ext_auth3_1 WHERE ikey > 20 UNION ALL SELECT bkey, fkey FROM db1_ext_auth3_2 WHERE ikey > 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth3_1 +POSTHOOK: Input: default@db1_ext_auth3_2 +#### A masked pattern was here #### +-15 65.0 +-16 66.0 +53 -455.454 +PREHOOK: query: DROP TABLE db1_ext_auth3_1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@db1_ext_auth3_1 +PREHOOK: Output: default@db1_ext_auth3_1 +POSTHOOK: query: DROP TABLE db1_ext_auth3_1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@db1_ext_auth3_1 +POSTHOOK: Output: default@db1_ext_auth3_1 +PREHOOK: query: DROP TABLE db1_ext_auth3_2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@db1_ext_auth3_2 +PREHOOK: Output: default@db1_ext_auth3_2 +POSTHOOK: query: DROP TABLE db1_ext_auth3_2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@db1_ext_auth3_2 +POSTHOOK: Output: default@db1_ext_auth3_2 +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0