diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index fe3630b952..8c4d9b7de7 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -527,6 +527,7 @@ minillaplocal.query.files=\ external_jdbc_table.q,\ external_jdbc_table2.q,\ external_jdbc_table3.q,\ + external_jdbc_table4.q,\ external_jdbc_table_partition.q,\ external_jdbc_table_typeconversion.q,\ fullouter_mapjoin_1_optimized.q,\ diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java index aabfd7c517..4d5ba6abf3 100644 --- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java +++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java @@ -88,7 +88,8 @@ public void initialize(Configuration conf, Properties properties) throws SerDeEx hiveColumnNames = fieldNamesProperty.trim().split(","); hiveColumnTypesList = TypeInfoUtils.getTypeInfosFromTypeString(fieldTypesProperty); } else { - hiveColumnNames = properties.getProperty(serdeConstants.LIST_COLUMNS).split(","); + // We need to get the column names from the accessor due to capitalization + hiveColumnNames = dbAccessor.getColumnNames(tableConfig).toArray(new String[0]); hiveColumnTypesList = TypeInfoUtils.getTypeInfosFromTypeString(properties.getProperty(serdeConstants.LIST_COLUMN_TYPES)); } if (hiveColumnNames.length == 0) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java index 00c3951489..5b21bf88e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelFactories.java @@ -37,10 +37,12 @@ import org.apache.calcite.rel.core.RelFactories.SemiJoinFactory; import org.apache.calcite.rel.core.RelFactories.SetOpFactory; import org.apache.calcite.rel.core.RelFactories.SortFactory; +import org.apache.calcite.rel.rel2sql.SqlImplementor; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate; @@ -101,7 +103,8 @@ private HiveRelFactories() { public RelNode createProject(RelNode child, List childExprs, List fieldNames) { RelOptCluster cluster = child.getCluster(); - RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), childExprs, fieldNames); + RelDataType rowType = RexUtil.createStructType( + cluster.getTypeFactory(), childExprs, fieldNames, SqlValidatorUtil.EXPR_SUGGESTER); RelTraitSet trait = TraitsUtil.getDefaultTraitSet(cluster, child.getTraitSet()); RelNode project = HiveProject.create(cluster, child, childExprs, rowType, trait, Collections. emptyList()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java index 2aafd1049b..e83ffe1217 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveTypeSystemImpl.java @@ -142,4 +142,10 @@ public int getMaxNumericScale() { public int getMaxNumericPrecision() { return MAX_DECIMAL_PRECISION; } + + @Override + public boolean isSchemaCaseSensitive() { + return false; + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java index f29b1f3c26..cfdc3eb9a7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveProject.java @@ -28,10 +28,12 @@ import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUtil; +import org.apache.calcite.sql.validate.SqlValidatorUtil; import org.apache.calcite.util.Util; import org.apache.calcite.util.mapping.Mapping; import org.apache.calcite.util.mapping.MappingType; @@ -88,7 +90,8 @@ public static HiveProject create(RelNode child, List exps, String msg = "Select list contains multiple expressions with the same name." + fieldNames; throw new CalciteSemanticException(msg, UnsupportedFeature.Same_name_in_multiple_expressions); } - RelDataType rowType = RexUtil.createStructType(cluster.getTypeFactory(), exps, fieldNames); + RelDataType rowType = RexUtil.createStructType( + cluster.getTypeFactory(), exps, fieldNames, SqlValidatorUtil.EXPR_SUGGESTER); return create(cluster, child, exps, rowType, Collections. emptyList()); } diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table4.q b/ql/src/test/queries/clientpositive/external_jdbc_table4.q new file mode 100644 index 0000000000..8145366797 --- /dev/null +++ b/ql/src/test/queries/clientpositive/external_jdbc_table4.q @@ -0,0 +1,139 @@ +--! qt:dataset:src + +-- SORT_QUERY_RESULTS + +CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'; + +FROM src +SELECT +dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;create=true','user1','passwd1', +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1; + +FROM src +SELECT +dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2;create=true','user2','passwd2', +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "BKEY" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1; + +FROM src +SELECT +dboutput ('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "FKEY" REAL, "DKEY" DOUBLE )' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','20','20','20.0','20.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1; + + + +CREATE EXTERNAL TABLE db1_ext_auth1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +); + +CREATE EXTERNAL TABLE db2_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user2", + "hive.sql.dbcp.password" = "passwd2", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +); + +CREATE EXTERNAL TABLE db1_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +); + + +SELECT * FROM db1_ext_auth1; + +SELECT * FROM db2_ext_auth2; + +SELECT * FROM db1_ext_auth2; + +EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey; + +SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey; + +EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey; + +SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey; + +EXPLAIN +SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b; + +SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b; + +DROP TABLE db1_ext_auth1; +DROP TABLE db2_ext_auth2; +DROP TABLE db1_ext_auth2; + +FROM src +SELECT +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth1','user1','passwd1', +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ), +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_auth2','user2','passwd2', +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1; diff --git a/ql/src/test/results/clientpositive/llap/external_jdbc_table4.q.out b/ql/src/test/results/clientpositive/llap/external_jdbc_table4.q.out new file mode 100644 index 0000000000..ece0a6f69a --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/external_jdbc_table4.q.out @@ -0,0 +1,584 @@ +PREHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: dboutput +POSTHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: dboutput +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1 ("IKEY","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "BKEY" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "BKEY" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","BKEY","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "FKEY" REAL, "DKEY" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'CREATE TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "FKEY" REAL, "DKEY" DOUBLE )' ), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','20','20','20.0','20.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +#### A masked pattern was here #### +'INSERT INTO EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2 ("ikey","bkey","FKEY","DKEY") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@db1_ext_auth1 +POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@db1_ext_auth1 +PREHOOK: query: CREATE EXTERNAL TABLE db2_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user2", + "hive.sql.dbcp.password" = "passwd2", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@db2_ext_auth2 +POSTHOOK: query: CREATE EXTERNAL TABLE db2_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user2", + "hive.sql.dbcp.password" = "passwd2", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@db2_ext_auth2 +PREHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@db1_ext_auth2 +POSTHOOK: query: CREATE EXTERNAL TABLE db1_ext_auth2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "user1", + "hive.sql.dbcp.password" = "passwd1", + "hive.sql.table" = "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@db1_ext_auth2 +PREHOOK: query: SELECT * FROM db1_ext_auth1 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +-20 -20 -20.0 -20.0 +100 -15 65.0 -74.0 +20 20 20.0 20.0 +44 53 -455.454 330.76 +PREHOOK: query: SELECT * FROM db2_ext_auth2 +PREHOOK: type: QUERY +PREHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db2_ext_auth2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +-20 8 9.0 11.0 +101 -16 66.0 -75.0 +20 20 20.0 20.0 +40 50 -455.4543 330.767 +PREHOOK: query: SELECT * FROM db1_ext_auth2 +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +-20 8 9.0 11.0 +101 -16 66.0 -75.0 +20 20 20.0 20.0 +40 50 -455.4543 330.767 +PREHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: db1_ext_auth1 + properties: + hive.sql.query SELECT "IKEY", "bkey", "fkey", "dkey" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1" +WHERE "IKEY" IS NOT NULL + hive.sql.query.fieldNames IKEY,bkey,fkey,dkey + hive.sql.query.fieldTypes int,bigint,float,double + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: float), _col3 (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: db2_ext_auth2 + properties: + hive.sql.query SELECT "ikey", "BKEY", "fkey", "dkey" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2" +WHERE "ikey" IS NOT NULL + hive.sql.query.fieldNames ikey,BKEY,fkey,dkey + hive.sql.query.fieldTypes int,bigint,float,double + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: bigint), _col2 (type: float), _col3 (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 _col0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + Statistics: Num rows: 1 Data size: 26 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 26 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth1 JOIN db2_ext_auth2 ON db1_ext_auth1.ikey = db2_ext_auth2.ikey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db2_ext_auth2 +#### A masked pattern was here #### +-20 -20 -20.0 -20.0 -20 8 9.0 11.0 +20 20 20.0 20.0 20 20 20.0 20.0 +PREHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: db1_ext_auth1 + properties: + hive.sql.query SELECT "t"."IKEY", "t"."bkey", "t"."fkey", "t"."dkey", "t0"."ikey" AS "ikey0", "t0"."bkey" AS "bkey0", "t0"."FKEY" AS "FKEY0", "t0"."DKEY" AS "DKEY0" +FROM (SELECT * +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1" +WHERE "IKEY" IS NOT NULL) AS "t" +INNER JOIN (SELECT * +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2" +WHERE "ikey" IS NOT NULL) AS "t0" ON "t"."IKEY" = "t0"."ikey" + hive.sql.query.fieldNames IKEY,bkey,fkey,dkey,ikey0,bkey0,FKEY0,DKEY0 + hive.sql.query.fieldTypes int,bigint,float,double,int,bigint,float,double + hive.sql.query.split false + Select Operator + expressions: ikey (type: int), bkey (type: bigint), fkey (type: float), dkey (type: double), ikey0 (type: int), bkey0 (type: bigint), fkey0 (type: float), dkey0 (type: double) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7 + ListSink + +PREHOOK: query: SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM db1_ext_auth1 JOIN db1_ext_auth2 ON db1_ext_auth1.ikey = db1_ext_auth2.ikey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Input: default@db1_ext_auth2 +#### A masked pattern was here #### +-20 -20 -20.0 -20.0 -20 8 9.0 11.0 +20 20 20.0 20.0 20 20 20.0 20.0 +Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: EXPLAIN +SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (XPROD_EDGE), Map 3 (XPROD_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: db1_ext_auth1 + properties: + hive.sql.query SELECT "IKEY" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1" + hive.sql.query.fieldNames IKEY + hive.sql.query.fieldTypes int + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: db1_ext_auth1 + properties: + hive.sql.query SELECT "IKEY" * 2 AS "*" +FROM "EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1" + hive.sql.query.fieldNames * + hive.sql.query.fieldTypes int + hive.sql.query.split true + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: * (type: int) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: int) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 + 1 + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 9 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +Warning: Shuffle Join MERGEJOIN[10][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b +PREHOOK: type: QUERY +PREHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT db1_ext_auth1.ikey, b.ikey * 2 FROM db1_ext_auth1 JOIN (SELECT * FROM db1_ext_auth1) b +POSTHOOK: type: QUERY +POSTHOOK: Input: default@db1_ext_auth1 +#### A masked pattern was here #### +-20 -40 +-20 200 +-20 40 +-20 88 +100 -40 +100 200 +100 40 +100 88 +20 -40 +20 200 +20 40 +20 88 +44 -40 +44 200 +44 40 +44 88 +PREHOOK: query: DROP TABLE db1_ext_auth1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@db1_ext_auth1 +PREHOOK: Output: default@db1_ext_auth1 +POSTHOOK: query: DROP TABLE db1_ext_auth1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@db1_ext_auth1 +POSTHOOK: Output: default@db1_ext_auth1 +PREHOOK: query: DROP TABLE db2_ext_auth2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@db2_ext_auth2 +PREHOOK: Output: default@db2_ext_auth2 +POSTHOOK: query: DROP TABLE db2_ext_auth2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@db2_ext_auth2 +POSTHOOK: Output: default@db2_ext_auth2 +PREHOOK: query: DROP TABLE db1_ext_auth2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@db1_ext_auth2 +PREHOOK: Output: default@db1_ext_auth2 +POSTHOOK: query: DROP TABLE db1_ext_auth2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@db1_ext_auth2 +POSTHOOK: Output: default@db1_ext_auth2 +PREHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src +SELECT +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE1' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ), +#### A masked pattern was here #### +'DROP TABLE EXTERNAL_JDBC_SIMPLE_DERBY2_TABLE2' ) +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0