Major Description
Caught this in 1.7 builds:
"PriorityRpcServer.handler=4,queue=0,port=42214" daemon prio=10 tid=0x00007f08d5786000 nid=0x2eed in Object.wait() [0x00007f08918d5000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.client.AsyncProcess.waitForMaximumCurrentTasks(AsyncProcess.java:1659) - locked <0x00000007580e5f98> (a java.util.concurrent.atomic.AtomicLong) at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1688) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:208) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.flush(BufferedMutatorImpl.java:183) - locked <0x00000007580f36f8> (a org.apache.hadoop.hbase.client.BufferedMutatorImpl) at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1430) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:1021) at org.apache.hadoop.hbase.security.access.AccessControlLists.addUserPermission(AccessControlLists.java:176) at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:2175) at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:2172) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614) at org.apache.hadoop.security.SecurityUtil.doAsUser(SecurityUtil.java:444) at org.apache.hadoop.security.SecurityUtil.doAsLoginUser(SecurityUtil.java:425) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.util.Methods.call(Methods.java:39) at org.apache.hadoop.hbase.security.User.runAsLoginUser(User.java:205) at org.apache.hadoop.hbase.security.access.AccessController.grant(AccessController.java:2172) at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService$1.grant(AccessControlProtos.java:9933) at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService.callMethod(AccessControlProtos.java:10097) at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7650) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:1896) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:1878) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32590) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2120) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=3,queue=1,port=42214" daemon prio=10 tid=0x00007f08d5784000 nid=0x2eec in Object.wait() [0x00007f08919d7000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1248) - locked <0x00000007cb61ecd8> (a org.apache.hadoop.hbase.ipc.Call) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:217) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:295) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.get(ClientProtos.java:33008) at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:862) at org.apache.hadoop.hbase.client.HTable$3.call(HTable.java:853) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:870) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:836) at org.apache.hadoop.hbase.security.access.AccessControlLists.getPermissions(AccessControlLists.java:464) at org.apache.hadoop.hbase.security.access.AccessController.updateACL(AccessController.java:268) at org.apache.hadoop.hbase.security.access.AccessController.postPut(AccessController.java:1615) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$32.call(RegionCoprocessorHost.java:956) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postPut(RegionCoprocessorHost.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3221) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2841) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2783) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:697) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:659) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2048) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32594) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2120) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=2,queue=0,port=42214" daemon prio=10 tid=0x00007f08d5782000 nid=0x2eeb in Object.wait() [0x00007f0891ad8000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.take(ResultBoundedCompletionService.java:148) - locked <0x00000007cc351438> (a [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:188) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:212) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:186) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1276) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1182) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1166) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1123) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getRegionLocation(ConnectionManager.java:958) at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:83) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:79) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:870) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:836) at org.apache.hadoop.hbase.security.access.AccessControlLists.getPermissions(AccessControlLists.java:464) at org.apache.hadoop.hbase.security.access.AccessController.updateACL(AccessController.java:268) at org.apache.hadoop.hbase.security.access.AccessController.postPut(AccessController.java:1615) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$32.call(RegionCoprocessorHost.java:956) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postPut(RegionCoprocessorHost.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3221) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2841) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2783) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:697) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:659) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2048) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32594) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2120) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=1,queue=1,port=42214" daemon prio=10 tid=0x00007f08d5780000 nid=0x2eea in Object.wait() [0x00007f0891bd8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.client.AsyncProcess.waitForMaximumCurrentTasks(AsyncProcess.java:1659) - locked <0x00000007581261c0> (a java.util.concurrent.atomic.AtomicLong) at org.apache.hadoop.hbase.client.AsyncProcess.waitForAllPreviousOpsAndReset(AsyncProcess.java:1688) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.backgroundFlushCommits(BufferedMutatorImpl.java:208) at org.apache.hadoop.hbase.client.BufferedMutatorImpl.flush(BufferedMutatorImpl.java:183) - locked <0x0000000758141d48> (a org.apache.hadoop.hbase.client.BufferedMutatorImpl) at org.apache.hadoop.hbase.client.HTable.flushCommits(HTable.java:1430) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:1021) at org.apache.hadoop.hbase.security.access.AccessControlLists.addUserPermission(AccessControlLists.java:176) at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:2175) at org.apache.hadoop.hbase.security.access.AccessController$8.run(AccessController.java:2172) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1614) at org.apache.hadoop.security.SecurityUtil.doAsUser(SecurityUtil.java:444) at org.apache.hadoop.security.SecurityUtil.doAsLoginUser(SecurityUtil.java:425) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.util.Methods.call(Methods.java:39) at org.apache.hadoop.hbase.security.User.runAsLoginUser(User.java:205) at org.apache.hadoop.hbase.security.access.AccessController.grant(AccessController.java:2172) at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService$1.grant(AccessControlProtos.java:9933) at org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos$AccessControlService.callMethod(AccessControlProtos.java:10097) at org.apache.hadoop.hbase.regionserver.HRegion.execService(HRegion.java:7650) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execServiceOnRegion(RSRpcServices.java:1896) at org.apache.hadoop.hbase.regionserver.RSRpcServices.execService(RSRpcServices.java:1878) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32590) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2120) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=0,queue=0,port=42214" daemon prio=10 tid=0x00007f08d577e000 nid=0x2ee8 in Object.wait() [0x00007f0891ddb000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService.take(ResultBoundedCompletionService.java:148) - locked <0x00000007ca626188> (a [Lorg.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture;) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:188) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:59) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.loadCache(ClientSmallReversedScanner.java:212) at org.apache.hadoop.hbase.client.ClientSmallReversedScanner.next(ClientSmallReversedScanner.java:186) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegionInMeta(ConnectionManager.java:1276) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1182) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1166) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.locateRegion(ConnectionManager.java:1123) at org.apache.hadoop.hbase.client.ConnectionManager$HConnectionImplementation.getRegionLocation(ConnectionManager.java:958) at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:83) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:79) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:124) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:870) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:836) at org.apache.hadoop.hbase.security.access.AccessControlLists.getPermissions(AccessControlLists.java:464) at org.apache.hadoop.hbase.security.access.AccessController.updateACL(AccessController.java:268) at org.apache.hadoop.hbase.security.access.AccessController.postPut(AccessController.java:1615) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$32.call(RegionCoprocessorHost.java:956) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost$RegionOperation.call(RegionCoprocessorHost.java:1673) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1749) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.execOperation(RegionCoprocessorHost.java:1705) at org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.postPut(RegionCoprocessorHost.java:952) at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3221) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2841) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2783) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:697) at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:659) at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:2048) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32594) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2120) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:106) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745)