Major Description
We are seeing ZK connection leaks when there's an error during Phoenix connection creation. ConnectionQueryServicesImpl grabs an HConnection during init, which creates a ZK ClientCnxn which starts two threads (EventThread, SendThread). Later in the Phoenix connection init, there's an exception (in our case, incorrect server jar version). Phoenix bubbles up the exception but never explicitly calls close on the HConnection, so the ZK threads stay alive.
This was perhaps partially by design as the HConnectionImplementation is supposed to have a DelayedClosing reaper thread that reaps any stale ZK connections. However, because of HBASE-11354, that reaper never gets started. (we are running HBase 0.98)
In any case, this reaper stuff was deprecated in HBASE-6778, so clients should close the connection themselves.
at org.apache.phoenix.exception.SQLExceptionCode$Factory$1.newException(SQLExceptionCode.java:422) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:145) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl.checkClientServerCompatibility(ConnectionQueryServicesImpl.java:1167) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:1034) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1370) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:2116) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:828) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.compile.CreateTableCompiler$2.execute(CreateTableCompiler.java:183) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:335) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:323) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:321) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1274) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2275) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl$13.call(ConnectionQueryServicesImpl.java:2244) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:78) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2244) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:233) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:135) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:202) ~[phoenix-core-4.7.0-sfdc-1.0.8.jar:4.7.0-sfdc-1.0.8] at java.sql.DriverManager.getConnection(DriverManager.java:664) ~[na:1.8.0_60] at java.sql.DriverManager.getConnection(DriverManager.java:270) ~[na:1.8.0_60]