[HIVE-9390] Enhance retry logic wrt DB access in TxnHandler

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 0.14.0
• Fix Version/s: 1.0.0
• Component/s: Transactions
• Assignee: Eugene Koifman
• Reporter: Eugene Koifman
• Created: 15/Jan/15 18:50
• Updated: 01/Oct/19 22:06
• Resolved: 17/Jan/15 02:55

Description

example of error

2015-01-13 16:09:21,148 ERROR metastore.RetryingHMSHandler (RetryingHMSHandler.java:invoke(141)) - org.apache.thrift.TException: MetaException(message:Unable to get jdbc connection from pool, Read timed out)

	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_open_txns(HiveMetaStore.java:5324)

	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)

	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

	at java.lang.reflect.Method.invoke(Method.java:606)

	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:102)

	at com.sun.proxy.$Proxy11.get_open_txns(Unknown Source)

	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getValidTxns(HiveMetaStoreClient.java:1696)

	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)

	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

	at java.lang.reflect.Method.invoke(Method.java:606)

	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:90)

	at com.sun.proxy.$Proxy12.getValidTxns(Unknown Source)

	at org.apache.hadoop.hive.ql.lockmgr.DbTxnManager.getValidTxns(DbTxnManager.java:289)

	at org.apache.hadoop.hive.ql.Driver.recordValidTxns(Driver.java:882)

	at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:399)

	at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:303)

	at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1067)

	at org.apache.hadoop.hive.ql.Driver.compileAndRespond(Driver.java:1061)

	at org.apache.hive.service.cli.operation.SQLOperation.prepare(SQLOperation.java:100)

	at org.apache.hive.service.cli.operation.SQLOperation.runInternal(SQLOperation.java:171)

	at org.apache.hive.service.cli.operation.Operation.run(Operation.java:256)

	at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementInternal(HiveSessionImpl.java:376)

	at org.apache.hive.service.cli.session.HiveSessionImpl.executeStatementAsync(HiveSessionImpl.java:363)

	at sun.reflect.GeneratedMethodAccessor27.invoke(Unknown Source)

	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

	at java.lang.reflect.Method.invoke(Method.java:606)

	at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:79)

	at org.apache.hive.service.cli.session.HiveSessionProxy.access$000(HiveSessionProxy.java:37)

	at org.apache.hive.service.cli.session.HiveSessionProxy$1.run(HiveSessionProxy.java:64)

	at java.security.AccessController.doPrivileged(Native Method)

	at javax.security.auth.Subject.doAs(Subject.java:415)

	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)

	at org.apache.hadoop.hive.shims.HadoopShimsSecure.doAs(HadoopShimsSecure.java:536)

	at org.apache.hive.service.cli.session.HiveSessionProxy.invoke(HiveSessionProxy.java:60)

	at com.sun.proxy.$Proxy21.executeStatementAsync(Unknown Source)

	at org.apache.hive.service.cli.CLIService.executeStatementAsync(CLIService.java:247)

	at org.apache.hive.service.cli.thrift.ThriftCLIService.ExecuteStatement(ThriftCLIService.java:401)

	at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1313)

	at org.apache.hive.service.cli.thrift.TCLIService$Processor$ExecuteStatement.getResult(TCLIService.java:1298)

	at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)

	at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)

	at org.apache.thrift.server.TServlet.doPost(TServlet.java:83)

	at org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:101)

	at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)

	at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)

	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565)

	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479)

	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225)

	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031)

	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406)

	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186)

	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965)

	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117)

	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111)

	at org.eclipse.jetty.server.Server.handle(Server.java:349)

	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449)

	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925)

	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857)

	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)

	at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:76)

	at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:609)

	at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:45)

	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

	at java.lang.Thread.run(Thread.java:745)

Caused by: MetaException(message:Unable to get jdbc connection from pool, Read timed out)

	at org.apache.hadoop.hive.metastore.txn.TxnHandler.getDbConn(TxnHandler.java:850)

	at org.apache.hadoop.hive.metastore.txn.TxnHandler.getOpenTxns(TxnHandler.java:196)

	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_open_txns(HiveMetaStore.java:5322)

	... 66 more

Issue Links

• is related to: HIVE-9404 NPE in org.apache.hadoop.hive.metastore.txn.TxnHandler.determineDatabaseProduct()
• is related to: HIVE-9938 Add retry logic to DbTxnMgr instead of aborting transactions.