From 40a6d9ee214ad4281441cd92d0297d8ca3fe5a25 Mon Sep 17 00:00:00 2001 From: murkrishn Date: Mon, 14 Sep 2015 23:51:39 -0700 Subject: [PATCH] In case Kylin is not able to serve result from cubes, the query is re-run in Hive Code and the result is returned on successful query execution. Apache Spark is used with Hive as the execution engine. Code updated with respect to https://issues.apache.org/jira/browse/KYLIN-742 --- .../java/org/apache/kylin/common/KylinConfig.java | 20 ++++++++ examples/test_case_data/sandbox/kylin.properties | 6 ++- .../apache/kylin/rest/service/QueryService.java | 56 +++++++++++++++++++++- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/kylin/common/KylinConfig.java b/common/src/main/java/org/apache/kylin/common/KylinConfig.java index d3220ee..7271d25 100644 --- a/common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -132,6 +132,12 @@ public class KylinConfig { public static final String HBASE_REGION_COUNT_MIN = "kylin.hbase.region.count.min"; public static final String HBASE_REGION_COUNT_MAX = "kylin.hbase.region.count.max"; + + // properties for JDBC Hive2 connection details + public static final String JDBC_HIVE2_URL = "kylin.jdbc.hive2.url"; + public static final String JDB_HIVE2_URL_DEFAULT = "jdbc:hive2://sandbox:10000"; + public static final String JDBC_HIVE2_USERNAME = "kylin.jdbc.hive2.username"; + public static final String JDBC_HIVE2_PASSWORD = "kylin.jdbc.hive2.password"; // static cached instances private static KylinConfig ENV_INSTANCE = null; @@ -265,6 +271,20 @@ public class KylinConfig { // ============================================================================ + // start: properties for JDBC Hive2 connection details + public String getJdbcHive2Url() { + return getOptional(JDBC_HIVE2_URL, "jdbc:hive2://sandbox:10000"); + } + + public String getJdbcHive2Username() { + return getOptional(JDBC_HIVE2_USERNAME, ""); + } + + public String getJdbcHive2Password() { + return getOptional(JDBC_HIVE2_PASSWORD, ""); + } + // end: properties for JDBC Hive2 connection details + public String getStorageUrl() { return storageUrl; } diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index b3ec4d0..8c703b4 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -129,5 +129,7 @@ ext.log.base.dir = /tmp/kylin_log1,/tmp/kylin_log2 #will set to kylin_query_log by default if not config here query.log.parse.result.table = kylin_query_log - - +## config for using hive on spark for querying ## +kylin.jdbc.hive2.url= +kylin.jdbc.hive2.username= +kylin.jdbc.hive2.password= diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java index 6e6d9da..37b4077 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -28,6 +28,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; +import java.sql.DriverManager; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -337,6 +338,8 @@ public class QueryService extends BasicService { List> results = new LinkedList>(); List columnMetas = new LinkedList(); + String driverName = "org.apache.hive.jdbc.HiveDriver"; + try { conn = getOLAPDataSource(sqlRequest.getProject()).getConnection(); @@ -372,7 +375,58 @@ public class QueryService extends BasicService { results.add(new LinkedList(oneRow)); oneRow.clear(); } - } finally { + } catch (SQLException sqlException) { + // unsuccessful statement execution, retry with Hive on Spark. Code modification as part of the jira https://issues.apache.org/jira/browse/KYLIN-742 + logger.error("exception in running the query: " + sql); + logger.debug("retrying query in hive"); + + try { + Class.forName(driverName); + } catch (ClassNotFoundException classNotFoundException) { + logger.error("exception in running the query on hive"); + classNotFoundException.printStackTrace(); + throw classNotFoundException; + } + + // configurations for connection to HiveServer2. + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); + String hiveServerUrl = kylinConfig.getJdbcHive2Url(); + String hiveUserName = kylinConfig.getJdbcHive2Username(); + String hivePassword = kylinConfig.getJdbcHive2Password(); + logger.info("connecting to : " + hiveServerUrl + " for executing the query"); + + try { + + conn = DriverManager.getConnection(hiveServerUrl, hiveUserName, hivePassword); + Statement stmt = conn.createStatement(); + resultSet = stmt.executeQuery(sql); + + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + + // Fill in selected column meta + for (int i = 1; i <= columnCount; ++i) { + columnMetas.add(new SelectedColumnMeta(metaData.isAutoIncrement(i), metaData.isCaseSensitive(i), false, metaData.isCurrency(i), metaData.isNullable(i), false, metaData.getColumnDisplaySize(i), metaData.getColumnLabel(i), metaData.getColumnName(i), null, null, null, metaData.getPrecision(i), metaData.getScale(i), metaData.getColumnType(i), metaData.getColumnTypeName(i), metaData.isReadOnly(i), false, false)); + } + + List oneRow = new LinkedList(); + + // fill in results. + while (resultSet.next()) { + //logger.debug("resultSet value: " + resultSet.getString(1)); + for (int i = 0; i < columnCount; i++) { + oneRow.add((resultSet.getString(i + 1))); + } + + results.add(new LinkedList(oneRow)); + oneRow.clear(); + } + } catch (SQLException sqlException1) { + logger.error("exception in running the query on hive"); + sqlException1.printStackTrace(); + throw sqlException1; + } + } finally { close(resultSet, stat, conn); } -- 1.9.1