diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 9e012ce..72d3f2e 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -519,6 +519,7 @@ minillaplocal.query.files=\ insert_values_orig_table_use_metadata.q,\ insert1_overwrite_partitions.q,\ jdbc_handler.q,\ + jdbc_handler2.q,\ join1.q,\ join_acid_non_acid.q,\ join_filters.q,\ diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java index f5472a0..2e0250d 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java @@ -15,6 +15,7 @@ package org.apache.hive.storage.jdbc; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; @@ -66,6 +67,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException LOGGER.trace("Initializing the SerDe"); if (tbl.containsKey(JdbcStorageConfig.DATABASE_TYPE.getPropertyName())) { + final boolean hiveQueryExecution = tbl.containsKey(Constants.HIVE_JDBC_QUERY); Configuration tableConfig = JdbcStorageConfigManager.convertPropertiesToConfiguration(tbl); @@ -73,17 +75,24 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException columnNames = dbAccessor.getColumnNames(tableConfig); numColumns = columnNames.size(); List hiveColumnNames; - - String[] hiveColumnNameArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMNS), ","); - if (numColumns != hiveColumnNameArray.length) { - throw new SerDeException("Expected " + numColumns + " columns. Table definition has " - + hiveColumnNameArray.length + " columns"); - } - hiveColumnNames = Arrays.asList(hiveColumnNameArray); - - hiveColumnTypeArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES), ":"); - if (hiveColumnTypeArray.length == 0) { - throw new SerDeException("Received an empty Hive column type definition"); + if (hiveQueryExecution) { + hiveColumnNames = columnNames; + final List columnTypes = dbAccessor.getColumnTypes(tableConfig); + hiveColumnTypeArray = new String[columnTypes.size()]; + hiveColumnTypeArray = columnTypes.toArray(hiveColumnTypeArray); + } else { + + String[] hiveColumnNameArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMNS), ","); + if (numColumns != hiveColumnNameArray.length) { + throw new SerDeException("Expected " + numColumns + " columns. Table definition has " + + hiveColumnNameArray.length + " columns"); + } + hiveColumnNames = Arrays.asList(hiveColumnNameArray); + + hiveColumnTypeArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES), ":"); + if (hiveColumnTypeArray.length == 0) { + throw new SerDeException("Received an empty Hive column type definition"); + } } List fieldInspectors = new ArrayList(numColumns); @@ -94,8 +103,8 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException } objectInspector = - ObjectInspectorFactory.getStandardStructObjectInspector(hiveColumnNames, - fieldInspectors); + ObjectInspectorFactory.getStandardStructObjectInspector(hiveColumnNames, + fieldInspectors); row = new ArrayList(numColumns); } } diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java index f2712b8..bff80a4 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java @@ -24,9 +24,11 @@ List getColumnNames(Configuration conf) throws HiveJdbcDatabaseAccessException; + List getColumnTypes(Configuration conf) throws HiveJdbcDatabaseAccessException; + int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException; JdbcRecordIterator - getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException; + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException; } diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java index af27c48..1409a48 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java @@ -35,6 +35,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -99,6 +100,69 @@ protected String getMetaDataQuery(Configuration conf) { } @Override + public List getColumnTypes(Configuration conf) throws HiveJdbcDatabaseAccessException { + Connection conn = null; + PreparedStatement ps = null; + ResultSet rs = null; + + try { + initializeDatabaseConnection(conf); + String metadataQuery = getMetaDataQuery(conf); + LOGGER.debug("Query to execute is [{}]", metadataQuery); + + conn = dbcpDataSource.getConnection(); + ps = conn.prepareStatement(metadataQuery); + rs = ps.executeQuery(); + + ResultSetMetaData metadata = rs.getMetaData(); + int numColumns = metadata.getColumnCount(); + List columnTypes = new ArrayList(numColumns); + for (int i = 0; i < numColumns; i++) { + switch (metadata.getColumnType(i + 1)) { + case Types.CHAR: + columnTypes.add(serdeConstants.STRING_TYPE_NAME); + break; + case Types.INTEGER: + columnTypes.add(serdeConstants.INT_TYPE_NAME); + break; + case Types.BIGINT: + columnTypes.add(serdeConstants.BIGINT_TYPE_NAME); + break; + case Types.DECIMAL: + columnTypes.add(serdeConstants.DECIMAL_TYPE_NAME); + break; + case Types.FLOAT: + case Types.REAL: + columnTypes.add(serdeConstants.FLOAT_TYPE_NAME); + break; + case Types.DOUBLE: + columnTypes.add(serdeConstants.DOUBLE_TYPE_NAME); + break; + case Types.DATE: + columnTypes.add(serdeConstants.DATE_TYPE_NAME); + break; + case Types.TIMESTAMP: + columnTypes.add(serdeConstants.TIMESTAMP_TYPE_NAME); + break; + + default: + columnTypes.add(metadata.getColumnTypeName(i+1)); + break; + } + } + + return columnTypes; + } catch (Exception e) { + LOGGER.error("Error while trying to get column names.", e); + throw new HiveJdbcDatabaseAccessException("Error while trying to get column names: " + e.getMessage(), e); + } finally { + cleanupResources(conn, ps, rs); + } + + } + + + @Override public int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException { Connection conn = null; PreparedStatement ps = null; @@ -136,7 +200,7 @@ public int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAc @Override public JdbcRecordIterator - getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException { + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException { Connection conn = null; PreparedStatement ps = null; diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java index d6c2736..ff552e2 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/JdbcRecordIterator.java @@ -47,7 +47,7 @@ public JdbcRecordIterator(Connection conn, PreparedStatement ps, ResultSet rs, S this.conn = conn; this.ps = ps; this.rs = rs; - if (typeString != null) { + if (false && typeString != null) { this.columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(typeString); } } diff --git ql/src/test/queries/clientpositive/external_jdbc_table.q ql/src/test/queries/clientpositive/external_jdbc_table.q new file mode 100644 index 0000000..13f4726 --- /dev/null +++ ql/src/test/queries/clientpositive/external_jdbc_table.q @@ -0,0 +1,186 @@ +--! qt:dataset:src + +set hive.strict.checks.cartesian.product= false; + + +CREATE TABLE simple_hive_table1 (ikey INT, bkey BIGINT, fkey FLOAT, dkey DOUBLE ); + +CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput'; + + +FROM src + +SELECT + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') + +limit 1; + +FROM src + +SELECT + +dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), + +dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') + +limit 1; + + +CREATE EXTERNAL TABLE ext_simple_derby_table1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +); + + +CREATE EXTERNAL TABLE ext_simple_derby_table2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", + "hive.sql.jdbc.url" = "jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true;collation=TERRITORY_BASED:PRIMARY", + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +); + + +select * from ext_simple_derby_table1; + +--Test projection +select dkey,fkey,bkey,ikey from ext_simple_derby_table1; +select bkey+ikey,fkey+dkey from ext_simple_derby_table1; +select abs(dkey),abs(ikey),abs(fkey),abs(bkey) from ext_simple_derby_table1; + + + +--Test aggregation +select count(*) from ext_simple_derby_table1; +select count(distinct bkey) from ext_simple_derby_table1; +select count(ikey), sum(bkey), avg(dkey), max(fkey) from ext_simple_derby_table1; + + +--Test sort +select dkey from ext_simple_derby_table1 order by dkey; +select SUM_IKEY,bkey from (select sum(-ikey) as SUM_IKEY, bkey from ext_simple_derby_table1 group by bkey) ttt order by bkey; + +--Test filter +explain select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey; +select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey; +select sum(bkey) from ext_simple_derby_table1 where ikey = 2450894 OR ikey = 2450911; +SELECT distinct dkey from ext_simple_derby_table1 where ikey = '100'; +SELECT count(*) FROM (select * from ext_simple_derby_table1) v WHERE ikey = 100; +SELECT count(*) from ext_simple_derby_table1 having count(*) > 0; +select sum(8),8 from ext_simple_derby_table1 where ikey = 1 group by 2; + + +--Test join +explain select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey); + +select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey); + + +explain select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey); + +select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey); + + +--Test union + +SELECT ikey FROM simple_hive_table1 +UNION +SELECT bkey FROM ext_simple_derby_table2; + + + + + + + + + + + + +----FAILURES---- + +--The following does not work due to invalid generated derby syntax: +--SELECT "dkey", COUNT("bkey") AS "$f1" FROM "SIMPLE_DERBY_TABLE1" GROUP BY "dkey" OFFSET 0 ROWS FETCH NEXT 10 ROWS ONLY {LIMIT 1} + +--SELECT dkey,count(bkey) from ext_simple_derby_table1 group by dkey limit 10; + + + + + +--Fails parse.CalcitePlanner: CBO failed, skipping CBO. +--select sum(fkey) from ext_simple_derby_table1 where bkey in (10, 100); + + + + +--Fails to ClassCastException +-- + + + + +--SELECT ikey FROM ext_simple_derby_table1 +--UNION +--SELECT bkey FROM ext_simple_derby_table2; + + + + + + + +--select dkey from ext_simple_derby_table1 order by dkey limit 10 offset 60; \ No newline at end of file diff --git ql/src/test/queries/clientpositive/jdbc_handler.q ql/src/test/queries/clientpositive/jdbc_handler.q index 4d7effd..d086735 100644 --- ql/src/test/queries/clientpositive/jdbc_handler.q +++ ql/src/test/queries/clientpositive/jdbc_handler.q @@ -21,7 +21,7 @@ limit 1; CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( diff --git ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out new file mode 100644 index 0000000..431e079 --- /dev/null +++ ql/src/test/results/clientpositive/llap/external_jdbc_table.q.out @@ -0,0 +1,533 @@ +PREHOOK: query: CREATE TABLE simple_hive_table1 (ikey INT, bkey BIGINT, fkey FLOAT, dkey DOUBLE ) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@simple_hive_table1 +POSTHOOK: query: CREATE TABLE simple_hive_table1 (ikey INT, bkey BIGINT, fkey FLOAT, dkey DOUBLE ) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@simple_hive_table1 +PREHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: dboutput +POSTHOOK: query: CREATE TEMPORARY FUNCTION dboutput AS 'org.apache.hadoop.hive.contrib.genericudf.example.GenericUDFDBOutput' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: dboutput +PREHOOK: query: FROM src + +SELECT + +#### A masked pattern was here #### +'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') + +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src + +SELECT + +#### A masked pattern was here #### +'CREATE TABLE SIMPLE_DERBY_TABLE1 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE)' ), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','-20','-20.0','-20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','100','-15','65.0','-74.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE1 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','44','53','-455.454','330.76') + +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: FROM src + +SELECT + +#### A masked pattern was here #### +'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') + +limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: FROM src + +SELECT + +#### A masked pattern was here #### +'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), + +#### A masked pattern was here #### +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') + +limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +0 0 0 0 0 +PREHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ext_simple_derby_table1 +POSTHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table1 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE1", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ext_simple_derby_table1 +PREHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ext_simple_derby_table2 +POSTHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table2 +( + ikey int, + bkey bigint, + fkey float, + dkey double +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( + "hive.sql.database.type" = "DERBY", + "hive.sql.jdbc.driver" = "org.apache.derby.jdbc.EmbeddedDriver", +#### A masked pattern was here #### + "hive.sql.dbcp.username" = "APP", + "hive.sql.dbcp.password" = "mine", + "hive.sql.table" = "SIMPLE_DERBY_TABLE2", + "hive.sql.dbcp.maxActive" = "1" +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ext_simple_derby_table2 +PREHOOK: query: select * from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +20 20 20.0 20.0 +-20 -20 -20.0 -20.0 +100 -15 65.0 -74.0 +44 53 -455.454 330.76 +PREHOOK: query: select dkey,fkey,bkey,ikey from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select dkey,fkey,bkey,ikey from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +20.0 20.0 20 20 +-20.0 -20.0 -20 -20 +-74.0 65.0 -15 100 +330.76 -455.454 53 44 +PREHOOK: query: select bkey+ikey,fkey+dkey from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select bkey+ikey,fkey+dkey from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +40 40.0 +-40 -40.0 +85 -9.0 +97 -124.69401000976563 +PREHOOK: query: select abs(dkey),abs(ikey),abs(fkey),abs(bkey) from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select abs(dkey),abs(ikey),abs(fkey),abs(bkey) from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +20.0 20 20.0 20 +20.0 20 20.0 20 +74.0 100 65.0 15 +330.76 44 455.454 53 +PREHOOK: query: select count(*) from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +4 +PREHOOK: query: select count(distinct bkey) from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select count(distinct bkey) from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +4 +PREHOOK: query: select count(ikey), sum(bkey), avg(dkey), max(fkey) from ext_simple_derby_table1 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select count(ikey), sum(bkey), avg(dkey), max(fkey) from ext_simple_derby_table1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +4 38 64.19 65.0 +PREHOOK: query: select dkey from ext_simple_derby_table1 order by dkey +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select dkey from ext_simple_derby_table1 order by dkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +-74.0 +-20.0 +20.0 +330.76 +PREHOOK: query: select SUM_IKEY,bkey from (select sum(-ikey) as SUM_IKEY, bkey from ext_simple_derby_table1 group by bkey) ttt order by bkey +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select SUM_IKEY,bkey from (select sum(-ikey) as SUM_IKEY, bkey from ext_simple_derby_table1 group by bkey) ttt order by bkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +20 -20 +-100 -15 +-20 20 +-44 53 +PREHOOK: query: explain select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey +PREHOOK: type: QUERY +POSTHOOK: query: explain select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: ext_simple_derby_table1 + properties: + hive.sql.generated.query SELECT "bkey" +FROM "SIMPLE_DERBY_TABLE1" +WHERE 100 < "ikey" + hive.sql.query SELECT "bkey" +FROM "SIMPLE_DERBY_TABLE1" +WHERE 100 < "ikey" + Select Operator + expressions: bkey (type: bigint) + outputColumnNames: _col0 + ListSink + +PREHOOK: query: select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select bkey from ext_simple_derby_table1 where 100 < ext_simple_derby_table1.ikey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +PREHOOK: query: select sum(bkey) from ext_simple_derby_table1 where ikey = 2450894 OR ikey = 2450911 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select sum(bkey) from ext_simple_derby_table1 where ikey = 2450894 OR ikey = 2450911 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +NULL +PREHOOK: query: SELECT distinct dkey from ext_simple_derby_table1 where ikey = '100' +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT distinct dkey from ext_simple_derby_table1 where ikey = '100' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +-74.0 +PREHOOK: query: SELECT count(*) FROM (select * from ext_simple_derby_table1) v WHERE ikey = 100 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) FROM (select * from ext_simple_derby_table1) v WHERE ikey = 100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +1 +PREHOOK: query: SELECT count(*) from ext_simple_derby_table1 having count(*) > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT count(*) from ext_simple_derby_table1 having count(*) > 0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +4 +PREHOOK: query: select sum(8),8 from ext_simple_derby_table1 where ikey = 1 group by 2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +POSTHOOK: query: select sum(8),8 from ext_simple_derby_table1 where ikey = 1 group by 2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +#### A masked pattern was here #### +PREHOOK: query: explain select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey) +PREHOOK: type: QUERY +POSTHOOK: query: explain select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: ext_simple_derby_table1 + properties: + hive.sql.generated.query SELECT "t"."fkey", "t0"."dkey" +FROM (SELECT * +FROM "SIMPLE_DERBY_TABLE1" +WHERE "ikey" IS NOT NULL) AS "t" +INNER JOIN (SELECT * +FROM "SIMPLE_DERBY_TABLE2" +WHERE "ikey" IS NOT NULL) AS "t0" ON "t"."ikey" = "t0"."ikey" + hive.sql.query SELECT "t"."fkey", "t0"."dkey" +FROM (SELECT * +FROM "SIMPLE_DERBY_TABLE1" +WHERE "ikey" IS NOT NULL) AS "t" +INNER JOIN (SELECT * +FROM "SIMPLE_DERBY_TABLE2" +WHERE "ikey" IS NOT NULL) AS "t0" ON "t"."ikey" = "t0"."ikey" + Select Operator + expressions: fkey (type: float), dkey (type: double) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey) +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table1 +PREHOOK: Input: default@ext_simple_derby_table2 +#### A masked pattern was here #### +POSTHOOK: query: select ext_simple_derby_table1.fkey, ext_simple_derby_table2.dkey from ext_simple_derby_table1 join ext_simple_derby_table2 on +(ext_simple_derby_table1.ikey = ext_simple_derby_table2.ikey) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table1 +POSTHOOK: Input: default@ext_simple_derby_table2 +#### A masked pattern was here #### +20.0 20.0 +-20.0 11.0 +PREHOOK: query: explain select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey) +PREHOOK: type: QUERY +POSTHOOK: query: explain select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: simple_hive_table1 + filterExpr: ikey is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ikey is not null (type: boolean) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ikey (type: int), fkey (type: float) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: int) + sort order: + + Map-reduce partition columns: _col0 (type: int) + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: float) + Execution mode: vectorized, llap + LLAP IO: no inputs + Map 3 + Map Operator Tree: + TableScan + alias: ext_simple_derby_table2 + properties: + hive.sql.generated.query SELECT * +FROM "SIMPLE_DERBY_TABLE2" +WHERE "ikey" IS NOT NULL + hive.sql.query SELECT * +FROM "SIMPLE_DERBY_TABLE2" +WHERE "ikey" IS NOT NULL + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: ikey (type: int) + sort order: + + Map-reduce partition columns: ikey (type: int) + Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE + value expressions: dkey (type: double) + Execution mode: vectorized, llap + LLAP IO: no inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: int) + 1 ikey (type: int) + outputColumnNames: _col1, _col5 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: float), _col5 (type: double) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey) +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table2 +PREHOOK: Input: default@simple_hive_table1 +#### A masked pattern was here #### +POSTHOOK: query: select simple_hive_table1.fkey, ext_simple_derby_table2.dkey from simple_hive_table1 join ext_simple_derby_table2 on +(simple_hive_table1.ikey = ext_simple_derby_table2.ikey) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table2 +POSTHOOK: Input: default@simple_hive_table1 +#### A masked pattern was here #### +PREHOOK: query: SELECT ikey FROM simple_hive_table1 +UNION +SELECT bkey FROM ext_simple_derby_table2 +PREHOOK: type: QUERY +PREHOOK: Input: default@ext_simple_derby_table2 +PREHOOK: Input: default@simple_hive_table1 +#### A masked pattern was here #### +POSTHOOK: query: SELECT ikey FROM simple_hive_table1 +UNION +SELECT bkey FROM ext_simple_derby_table2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ext_simple_derby_table2 +POSTHOOK: Input: default@simple_hive_table1 +#### A masked pattern was here #### +8 +-16 +20 +50 diff --git ql/src/test/results/clientpositive/llap/jdbc_handler.q.out ql/src/test/results/clientpositive/llap/jdbc_handler.q.out index f55bda5..ade8d46 100644 --- ql/src/test/results/clientpositive/llap/jdbc_handler.q.out +++ ql/src/test/results/clientpositive/llap/jdbc_handler.q.out @@ -37,7 +37,7 @@ POSTHOOK: Input: default@src 0 0 0 PREHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( @@ -54,7 +54,7 @@ PREHOOK: Output: database:default PREHOOK: Output: default@ext_simple_derby_table POSTHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( @@ -101,7 +101,7 @@ WHERE 100 < "kkey" FROM "SIMPLE_DERBY_TABLE" WHERE 100 < "kkey" Select Operator - expressions: kkey (type: bigint) + expressions: kkey (type: int) outputColumnNames: _col0 ListSink