diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java index f5472a0..2e0250d 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java @@ -15,6 +15,7 @@ package org.apache.hive.storage.jdbc; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.AbstractSerDe; import org.apache.hadoop.hive.serde2.SerDeException; @@ -66,6 +67,7 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException LOGGER.trace("Initializing the SerDe"); if (tbl.containsKey(JdbcStorageConfig.DATABASE_TYPE.getPropertyName())) { + final boolean hiveQueryExecution = tbl.containsKey(Constants.HIVE_JDBC_QUERY); Configuration tableConfig = JdbcStorageConfigManager.convertPropertiesToConfiguration(tbl); @@ -73,17 +75,24 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException columnNames = dbAccessor.getColumnNames(tableConfig); numColumns = columnNames.size(); List hiveColumnNames; - - String[] hiveColumnNameArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMNS), ","); - if (numColumns != hiveColumnNameArray.length) { - throw new SerDeException("Expected " + numColumns + " columns. Table definition has " - + hiveColumnNameArray.length + " columns"); - } - hiveColumnNames = Arrays.asList(hiveColumnNameArray); - - hiveColumnTypeArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES), ":"); - if (hiveColumnTypeArray.length == 0) { - throw new SerDeException("Received an empty Hive column type definition"); + if (hiveQueryExecution) { + hiveColumnNames = columnNames; + final List columnTypes = dbAccessor.getColumnTypes(tableConfig); + hiveColumnTypeArray = new String[columnTypes.size()]; + hiveColumnTypeArray = columnTypes.toArray(hiveColumnTypeArray); + } else { + + String[] hiveColumnNameArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMNS), ","); + if (numColumns != hiveColumnNameArray.length) { + throw new SerDeException("Expected " + numColumns + " columns. Table definition has " + + hiveColumnNameArray.length + " columns"); + } + hiveColumnNames = Arrays.asList(hiveColumnNameArray); + + hiveColumnTypeArray = parseProperty(tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES), ":"); + if (hiveColumnTypeArray.length == 0) { + throw new SerDeException("Received an empty Hive column type definition"); + } } List fieldInspectors = new ArrayList(numColumns); @@ -94,8 +103,8 @@ public void initialize(Configuration conf, Properties tbl) throws SerDeException } objectInspector = - ObjectInspectorFactory.getStandardStructObjectInspector(hiveColumnNames, - fieldInspectors); + ObjectInspectorFactory.getStandardStructObjectInspector(hiveColumnNames, + fieldInspectors); row = new ArrayList(numColumns); } } diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java index f2712b8..bff80a4 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/DatabaseAccessor.java @@ -24,9 +24,11 @@ List getColumnNames(Configuration conf) throws HiveJdbcDatabaseAccessException; + List getColumnTypes(Configuration conf) throws HiveJdbcDatabaseAccessException; + int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException; JdbcRecordIterator - getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException; + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException; } diff --git jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java index af27c48..1409a48 100644 --- jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java +++ jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessor.java @@ -35,6 +35,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -99,6 +100,69 @@ protected String getMetaDataQuery(Configuration conf) { } @Override + public List getColumnTypes(Configuration conf) throws HiveJdbcDatabaseAccessException { + Connection conn = null; + PreparedStatement ps = null; + ResultSet rs = null; + + try { + initializeDatabaseConnection(conf); + String metadataQuery = getMetaDataQuery(conf); + LOGGER.debug("Query to execute is [{}]", metadataQuery); + + conn = dbcpDataSource.getConnection(); + ps = conn.prepareStatement(metadataQuery); + rs = ps.executeQuery(); + + ResultSetMetaData metadata = rs.getMetaData(); + int numColumns = metadata.getColumnCount(); + List columnTypes = new ArrayList(numColumns); + for (int i = 0; i < numColumns; i++) { + switch (metadata.getColumnType(i + 1)) { + case Types.CHAR: + columnTypes.add(serdeConstants.STRING_TYPE_NAME); + break; + case Types.INTEGER: + columnTypes.add(serdeConstants.INT_TYPE_NAME); + break; + case Types.BIGINT: + columnTypes.add(serdeConstants.BIGINT_TYPE_NAME); + break; + case Types.DECIMAL: + columnTypes.add(serdeConstants.DECIMAL_TYPE_NAME); + break; + case Types.FLOAT: + case Types.REAL: + columnTypes.add(serdeConstants.FLOAT_TYPE_NAME); + break; + case Types.DOUBLE: + columnTypes.add(serdeConstants.DOUBLE_TYPE_NAME); + break; + case Types.DATE: + columnTypes.add(serdeConstants.DATE_TYPE_NAME); + break; + case Types.TIMESTAMP: + columnTypes.add(serdeConstants.TIMESTAMP_TYPE_NAME); + break; + + default: + columnTypes.add(metadata.getColumnTypeName(i+1)); + break; + } + } + + return columnTypes; + } catch (Exception e) { + LOGGER.error("Error while trying to get column names.", e); + throw new HiveJdbcDatabaseAccessException("Error while trying to get column names: " + e.getMessage(), e); + } finally { + cleanupResources(conn, ps, rs); + } + + } + + + @Override public int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAccessException { Connection conn = null; PreparedStatement ps = null; @@ -136,7 +200,7 @@ public int getTotalNumberOfRecords(Configuration conf) throws HiveJdbcDatabaseAc @Override public JdbcRecordIterator - getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException { + getRecordIterator(Configuration conf, int limit, int offset) throws HiveJdbcDatabaseAccessException { Connection conn = null; PreparedStatement ps = null; diff --git ql/src/test/queries/clientpositive/jdbc_handler.q ql/src/test/queries/clientpositive/jdbc_handler.q index 4d7effd..d086735 100644 --- ql/src/test/queries/clientpositive/jdbc_handler.q +++ ql/src/test/queries/clientpositive/jdbc_handler.q @@ -21,7 +21,7 @@ limit 1; CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( diff --git ql/src/test/results/clientpositive/llap/jdbc_handler.q.out ql/src/test/results/clientpositive/llap/jdbc_handler.q.out index f55bda5..ade8d46 100644 --- ql/src/test/results/clientpositive/llap/jdbc_handler.q.out +++ ql/src/test/results/clientpositive/llap/jdbc_handler.q.out @@ -37,7 +37,7 @@ POSTHOOK: Input: default@src 0 0 0 PREHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( @@ -54,7 +54,7 @@ PREHOOK: Output: database:default PREHOOK: Output: default@ext_simple_derby_table POSTHOOK: query: CREATE EXTERNAL TABLE ext_simple_derby_table ( - kkey bigint + kkey int ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( @@ -101,7 +101,7 @@ WHERE 100 < "kkey" FROM "SIMPLE_DERBY_TABLE" WHERE 100 < "kkey" Select Operator - expressions: kkey (type: bigint) + expressions: kkey (type: int) outputColumnNames: _col0 ListSink