diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d62e527..63aadaf 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2595,7 +2595,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal // TODO: Make use of this config to configure fetch size HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE("hive.server2.thrift.resultset.max.fetch.size", 10000, "Max number of rows sent in one Fetch RPC call by the server to the client."), - HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE("hive.server2.resultset.default.fetch.size", 10000, + HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE("hive.server2.thrift.resultset.default.fetch.size", 10000, "The number of rows sent in one Fetch RPC call by the server to the client, if not\n" + "specified by the client."), HIVE_SERVER2_XSRF_FILTER_ENABLED("hive.server2.xsrf.filter.enabled",false, @@ -4157,6 +4157,7 @@ private static String getSQLStdAuthDefaultWhiteListPattern() { * sql standard authorization enabled */ static final String [] sqlStdAuthSafeVarNameRegexes = new String [] { + "fetchSize", "hive\\.auto\\..*", "hive\\.cbo\\..*", "hive\\.convert\\..*", diff --git jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index d6cf744..8c50242 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -637,6 +637,8 @@ private void openSession() throws SQLException { } // switch the database openConf.put("use:database", connParams.getDbName()); + // set the fetchSize + openConf.put("hive.server2.thrift.resultset.default.fetch.size", Integer.toString(fetchSize)); // set the session configuration Map sessVars = connParams.getSessionVars(); diff --git serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftJDBCBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftJDBCBinarySerDe.java index 84ed6ba..943c1e5 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftJDBCBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/thrift/ThriftJDBCBinarySerDe.java @@ -71,7 +71,9 @@ @Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { // Get column names - MAX_BUFFERED_ROWS = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE); + MAX_BUFFERED_ROWS = + HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE); + LOG.info("ThriftJDBCBinarySerDe max fetch size: " + MAX_BUFFERED_ROWS); String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS); String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES); final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl diff --git service/src/java/org/apache/hive/service/cli/CLIService.java service/src/java/org/apache/hive/service/cli/CLIService.java index de44ecb..344e3cc 100644 --- service/src/java/org/apache/hive/service/cli/CLIService.java +++ service/src/java/org/apache/hive/service/cli/CLIService.java @@ -78,7 +78,7 @@ public CLIService(HiveServer2 hiveServer2) { public synchronized void init(HiveConf hiveConf) { this.hiveConf = hiveConf; sessionManager = new SessionManager(hiveServer2); - defaultFetchRows = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE); + defaultFetchRows = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE); addService(sessionManager); // If the hadoop cluster is secure, do a kerberos login for the service from the keytab if (UserGroupInformation.isSecurityEnabled()) { diff --git service/src/java/org/apache/hive/service/cli/CLIServiceClient.java service/src/java/org/apache/hive/service/cli/CLIServiceClient.java index 4b84872..c965abc 100644 --- service/src/java/org/apache/hive/service/cli/CLIServiceClient.java +++ service/src/java/org/apache/hive/service/cli/CLIServiceClient.java @@ -31,10 +31,10 @@ * */ public abstract class CLIServiceClient implements ICLIService { - protected int defaultFetchRows = ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal; + protected int defaultFetchRows = ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.defaultIntVal; public CLIServiceClient(Configuration conf) { - defaultFetchRows = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_RESULTSET_DEFAULT_FETCH_SIZE); + defaultFetchRows = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE); } public SessionHandle openSession(String username, String password) diff --git service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index f939a93..1cf8fc4 100644 --- service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -455,6 +455,9 @@ public GetInfoValue getInfo(GetInfoType getInfoType) return new GetInfoValue(128); case CLI_MAX_TABLE_NAME_LEN: return new GetInfoValue(128); + case CLI_MAX_ROW_SIZE: + return new GetInfoValue(Integer.toString(sessionConf.getIntVar( + HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE))); case CLI_TXN_CAPABLE: default: throw new HiveSQLException("Unrecognized GetInfoType value: " + getInfoType.toString()); diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 2938338..42a907f 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -309,6 +309,20 @@ public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException { LOG.info("Client protocol version: " + req.getClient_protocol()); TOpenSessionResp resp = new TOpenSessionResp(); try { + Map openConf = req.getConfiguration(); + + // Set fetch size in hive conf + int maxFetchSize = + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE); + String confFetchSize = openConf != null ? + openConf.get(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname) : null; + if (confFetchSize != null && !confFetchSize.isEmpty()) { + int fetchSize = Integer.parseInt(confFetchSize); + openConf.put( + HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname, + Integer.toString(fetchSize > maxFetchSize ? maxFetchSize : fetchSize)); + } + SessionHandle sessionHandle = getSessionHandle(req, resp); resp.setSessionHandle(sessionHandle.toTSessionHandle()); // TODO: set real configuration map @@ -697,6 +711,12 @@ public TGetResultSetMetadataResp GetResultSetMetadata(TGetResultSetMetadataReq r public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException { TFetchResultsResp resp = new TFetchResultsResp(); try { + // Set fetch size + int maxFetchSize = + hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE); + if (req.getMaxRows() > maxFetchSize) { + req.setMaxRows(maxFetchSize); + } RowSet rowSet = cliService.fetchResults( new OperationHandle(req.getOperationHandle()), FetchOrientation.getFetchOrientation(req.getOrientation()), diff --git service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java index abb1ecf..3f163ec 100644 --- service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java +++ service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hive.service.Service; +import org.apache.hive.service.cli.GetInfoType; +import org.apache.hive.service.cli.GetInfoValue; import org.apache.hive.service.cli.OperationHandle; import org.apache.hive.service.cli.OperationState; import org.apache.hive.service.cli.OperationStatus; @@ -133,6 +135,42 @@ public void testOpenSession() throws Exception { } @Test + public void testFetchSize() throws Exception { + Map configuration = new HashMap(); + // Test fetchSize under max + configuration.put("hive.server2.thrift.resultset.default.fetch.size", "50"); + // Open a new client session + SessionHandle sessHandle = client.openSession(USERNAME, + PASSWORD, configuration); + // Session handle should not be null + assertNotNull("Session handle should not be null", sessHandle); + // Fetch Size should be set correctly on the server + GetInfoValue infoValue = client.getInfo(sessHandle, GetInfoType.CLI_MAX_ROW_SIZE); + assertEquals("50", infoValue.getStringValue()); + // Close client session + client.closeSession(sessHandle); + + // Test fetchSize over max + configuration.put( + "hive.server2.thrift.resultset.default.fetch.size", + Integer.toString(hiveServer2.getHiveConf().getIntVar( + HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE) + 1)); + // Open a new client session + sessHandle = client.openSession(USERNAME, + PASSWORD, configuration); + // Session handle should not be null + assertNotNull("Session handle should not be null", sessHandle); + // Fetch Size should be set to the max fetch size + infoValue = client.getInfo(sessHandle, GetInfoType.CLI_MAX_ROW_SIZE); + assertEquals( + Integer.toString(hiveServer2.getHiveConf() + .getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_MAX_FETCH_SIZE)), + infoValue.getStringValue()); + // Close client session + client.closeSession(sessHandle); + } + + @Test public void testGetFunctions() throws Exception { SessionHandle sessHandle = client.openSession(USERNAME, PASSWORD, new HashMap());