diff --git beeline/src/java/org/apache/hive/beeline/BeeLine.java beeline/src/java/org/apache/hive/beeline/BeeLine.java index dc79712054..491fe7bd9f 100644 --- beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -175,6 +175,7 @@ private static final String HIVE_CONF_PREFIX = "--hiveconf"; private static final String PROP_FILE_PREFIX = "--property-file"; public static final String PASSWD_MASK = "[passwd stripped]"; + public int fetchSize = -1; private final Map formats = map(new Object[] { "vertical", new VerticalOutputFormat(this), @@ -434,6 +435,13 @@ static Manifest getManifest() throws IOException { return null; } + public void setFetchSize(int fetchRows) { + if (fetchRows <= 0) { // if set to 0 or less, use server side default + this.fetchSize = Utils.getServerDefaultFetchSize(); + } else { + this.fetchSize = fetchRows; + } + } String getManifestAttribute(String name) { try { @@ -2341,12 +2349,16 @@ int print(ResultSet rs) throws SQLException { Statement createStatement() throws SQLException { Statement stmnt = getDatabaseConnection().getConnection().createStatement(); + if (getOpts().timeout > -1) { stmnt.setQueryTimeout(getOpts().timeout); } if (signalHandler != null) { signalHandler.setStatement(stmnt); } + if (fetchSize > -1) { + stmnt.setFetchSize(fetchSize); + } return stmnt; } diff --git beeline/src/java/org/apache/hive/beeline/Commands.java beeline/src/java/org/apache/hive/beeline/Commands.java index 8f47323700..665384761d 100644 --- beeline/src/java/org/apache/hive/beeline/Commands.java +++ beeline/src/java/org/apache/hive/beeline/Commands.java @@ -52,6 +52,8 @@ import java.util.Properties; import java.util.Set; import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; import org.apache.hadoop.hive.conf.HiveConf; @@ -1051,6 +1053,17 @@ private boolean executeInternal(String sql, boolean call) { true, beeLine.getErrorStream()); } } + // anytime this property is set, reset the local value so it can be set on new Statement + Pattern setPattern = Pattern.compile("(\\s*)(set)(\\s+)(\\S+)(\\s*)=(\\s*)(-?\\d+)(\\s*)(;*)"); + Matcher matcher = setPattern.matcher(sql.toLowerCase()); + + if (matcher.find() && matcher.group(4).equals("hive.server2.thrift.resultset.default.fetch.size")) { + try { + beeLine.setFetchSize(Integer.parseInt(matcher.group(7))); + } catch (NumberFormatException nfe) { + // ignore + } + } } finally { if (logThread != null) { if (!logThread.isInterrupted()) { diff --git jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index cb0b0d1c92..720c9fa957 100644 --- jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -146,7 +146,7 @@ private final List supportedProtocols = new LinkedList(); private int loginTimeout = 0; private TProtocolVersion protocol; - private int fetchSize = HiveStatement.DEFAULT_FETCH_SIZE; + private int fetchSize = 0; // default to server value private String initFile = null; private String wmPool = null, wmApp = null; private Properties clientInfo; @@ -838,6 +838,7 @@ private void openSession() throws SQLException { // set the fetchSize openConf.put("set:hiveconf:hive.server2.thrift.resultset.default.fetch.size", Integer.toString(fetchSize)); + if (wmPool != null) { openConf.put("set:hivevar:wmpool", wmPool); } @@ -874,7 +875,9 @@ private void openSession() throws SQLException { String serverFetchSize = openResp.getConfiguration().get("hive.server2.thrift.resultset.default.fetch.size"); if (serverFetchSize != null) { - fetchSize = Integer.parseInt(serverFetchSize); + Utils.setServerDefaultFetchSize(Integer.parseInt(serverFetchSize)); + if (fetchSize <= 0) // use server-side value when either set to 0 or not set at all (-1) + fetchSize = Utils.getServerDefaultFetchSize(); } } catch (TException e) { LOG.error("Error opening session", e); diff --git jdbc/src/java/org/apache/hive/jdbc/Utils.java jdbc/src/java/org/apache/hive/jdbc/Utils.java index e23826eb56..db45931358 100644 --- jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -68,6 +68,7 @@ static final String HIVE_SERVER2_RETRY_KEY = "hive.server2.retryserver"; static final String HIVE_SERVER2_RETRY_TRUE = "true"; static final String HIVE_SERVER2_RETRY_FALSE = "false"; + static int serverDefaultFetchSize = -1; public static class JdbcConnectionParams { // Note on client side parameter naming convention: @@ -718,4 +719,16 @@ public static String getCanonicalHostName(String hostName) { } } + public static int getServerDefaultFetchSize() { + if (serverDefaultFetchSize > 0) + return serverDefaultFetchSize; + else + return HiveStatement.DEFAULT_FETCH_SIZE; // protection against server side being set incorrectly. + } + + public static void setServerDefaultFetchSize(int size) { + if (size > 0) { + serverDefaultFetchSize = size; + } + } } diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 8f13fb3cca..93b86541b1 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -324,13 +324,11 @@ public TOpenSessionResp OpenSession(TOpenSessionReq req) throws TException { SessionHandle sessionHandle = getSessionHandle(req, resp); resp.setSessionHandle(sessionHandle.toTSessionHandle()); Map configurationMap = new HashMap(); - // Set the updated fetch size from the server into the configuration map for the client + // client sends its value to be stored in the sessionConf, server sends its value for the client to default to. HiveConf sessionConf = cliService.getSessionConf(sessionHandle); configurationMap.put( HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE.varname, - Integer.toString(sessionConf != null ? - sessionConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE) : - hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE))); + Integer.toString(hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE))); resp.setConfiguration(configurationMap); resp.setStatus(OK_STATUS); ThriftCLIServerContext context =