[PHOENIX-4474] Found some hanging tests - ASF JIRA

XML

Word

Printable

JSON

Details

Type: Sub-task
Status: Resolved
Priority: Major
Resolution: Fixed
Affects Version/s: None
Fix Version/s: 5.0.0-alpha
Component/s: None
Labels:
- HBase-2.0

Description

ExplainPlanWithStatsDisabledIT
ConcurrentMutationsIT
NumericArithmeticIT
AggregateQueryIT

AggregateQueryIT


Mon Dec 18 23:49:20 IST 2017, RpcRetryingCaller{globalStartTime=1513621155916, pause=100, maxAttempts=7}, java.net.ConnectException: Call to /192.168.1.3:56675 failed on connection exception: org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /192.168.1.3:56675

	at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:145)
	at org.apache.hadoop.hbase.client.HTable.get(HTable.java:388)
	at org.apache.hadoop.hbase.client.HTable.get(HTable.java:362)
	at org.apache.hadoop.hbase.MetaTableAccessor.getTableState(MetaTableAccessor.java:1118)
	at org.apache.hadoop.hbase.master.TableStateManager.readMetaState(TableStateManager.java:190)
	at org.apache.hadoop.hbase.master.TableStateManager.isTablePresent(TableStateManager.java:147)
	at org.apache.hadoop.hbase.master.HMaster.getTableDescriptors(HMaster.java:3135)
	at org.apache.hadoop.hbase.master.HMaster.listTableDescriptors(HMaster.java:3079)
	at org.apache.hadoop.hbase.master.MasterRpcServices.getTableDescriptors(MasterRpcServices.java:999)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:403)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:130)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:325)
	at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:305)
Caused by: java.net.ConnectException: Call to /192.168.1.3:56675 failed on connection exception: org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /192.168.1.3:56675
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:165)
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:390)
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:95)
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:410)
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:406)
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:103)
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:118)
	at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:307)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1352)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:329)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:315)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:920)
	at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179)
	at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71)
	at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:269)
	at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:263)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:500)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:479)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:122)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:327)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:343)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:632)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:579)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:496)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
	at org.apache.hadoop.hbase.shaded.io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.hadoop.hbase.shaded.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /192.168.1.3:56675
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:352)
	at org.apache.hadoop.hbase.shaded.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:340)
	... 7 more
Caused by: java.net.ConnectException: Connection refused
	... 11 more


[ERROR] Tests run: 7, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 705.751 s <<< FAILURE! - in org.apache.phoenix.end2end.NativeHBaseTypesIT
[ERROR] testRangeQuery1(org.apache.phoenix.end2end.NativeHBaseTypesIT)  Time elapsed: 610.63 s  <<< ERROR!
org.apache.phoenix.exception.PhoenixIOException: java.util.concurrent.TimeoutException: The procedure 37 is still running
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154)
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162)
Caused by: org.apache.hadoop.hbase.exceptions.TimeoutIOException: java.util.concurrent.TimeoutException: The procedure 37 is still running
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154)
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162)
Caused by: java.util.concurrent.TimeoutException: The procedure 37 is still running
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.initTableValues(NativeHBaseTypesIT.java:154)
	at org.apache.phoenix.end2end.NativeHBaseTypesIT.testRangeQuery1(NativeHBaseTypesIT.java:162)

NumericArithmeticIT

java.io.IOException: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64
	at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:159)
	at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:116)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure$1.createHdfsRegions(CreateTableProcedure.java:290)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:310)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:283)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:100)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:52)
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:182)
	at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:845)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1455)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1224)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:77)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1734)
Caused by: java.util.concurrent.ExecutionException: java.io.IOException: The specified region already exists on disk: hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegions(ModifyRegionUtils.java:153)
	... 12 more
Caused by: java.io.IOException: The specified region already exists on disk: hdfs://localhost:57532/user/asinghal/test-data/0c88fe5d-8cbc-495c-9668-04ab6847b35f/.tmp/data/default/T000365/d92d9658645e600021fc351c4af8ba64
	at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.createRegionOnFileSystem(HRegionFileSystem.java:975)
	at org.apache.hadoop.hbase.regionserver.HRegion.createHRegion(HRegion.java:6776)
	at org.apache.hadoop.hbase.util.ModifyRegionUtils.createRegion(ModifyRegionUtils.java:181)
	at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:146)
	at org.apache.hadoop.hbase.util.ModifyRegionUtils$1.call(ModifyRegionUtils.java:143)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)


2017-12-19 00:40:17,051 FATAL [ProcExecWrkr-15] org.apache.hadoop.hbase.procedure2.ProcedureExecutor(1367): CODE-BUG: Uncaught runtime exception for pid=137, state=FAILED:CREATE_TABLE_WRITE_FS_LAYOUT, exception=java.lang.OutOfMemoryError via CODE-BUG: Uncaught runtime exception: pid=137, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT; CreateTableProcedure table=T000360:java.lang.OutOfMemoryError: unable to create new native thread; CreateTableProcedure table=T000360
java.lang.UnsupportedOperationException: unhandled state=CREATE_TABLE_WRITE_FS_LAYOUT
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.rollbackState(CreateTableProcedure.java:146)
	at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.rollbackState(CreateTableProcedure.java:52)
	at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.rollback(StateMachineProcedure.java:201)
	at org.apache.hadoop.hbase.procedure2.Procedure.doRollback(Procedure.java:859)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1356)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1312)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1181)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:77)
	at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1734)

Attachments

Activity

People

Assignee:: Unassigned

Reporter:: Ankit Singhal

Votes:: 0 Vote for this issue

Watchers:: 1 Start watching this issue

Dates

Created:: 19/Dec/17 12:06

Updated:: 27/Dec/17 05:27

Resolved:: 27/Dec/17 05:27