Error Message Failed after attempts=36, exceptions: Thu Feb 23 21:38:12 UTC 2017, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=68077: row 'scantest0,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=xxx,36349,1487883585250, seqNum=0 Stacktrace org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Thu Feb 23 21:38:12 UTC 2017, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=68077: row 'scantest0,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=xxx,36349,1487883585250, seqNum=0 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:271) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:210) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:60) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:327) at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:302) at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:167) at org.apache.hadoop.hbase.client.ClientScanner.(ClientScanner.java:162) at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:794) at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:193) at org.apache.hadoop.hbase.client.MetaScanner.metaScan(MetaScanner.java:89) at org.apache.hadoop.hbase.client.MetaScanner.allTableRegions(MetaScanner.java:324) at org.apache.hadoop.hbase.client.HRegionLocator.getAllRegionLocations(HRegionLocator.java:89) at org.apache.hadoop.hbase.util.RegionSizeCalculator.init(RegionSizeCalculator.java:94) at org.apache.hadoop.hbase.util.RegionSizeCalculator.(RegionSizeCalculator.java:81) at org.apache.hadoop.hbase.mapreduce.MultiTableInputFormatBase.getSplits(MultiTableInputFormatBase.java:162) at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:301) at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:318) at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:196) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290) at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287) at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308) at org.apache.hadoop.hbase.mapreduce.TestMultiTableInputFormat.testScan(TestMultiTableInputFormat.java:247) at org.apache.hadoop.hbase.mapreduce.TestMultiTableInputFormat.testScanOBBToOPP(TestMultiTableInputFormat.java:192) Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=68077: row 'scantest0,,00000000000000' on table 'hbase:meta' at region=hbase:meta,,1.1588230740, hostname=xxx,36349,1487883585250, seqNum=0 at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722) Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:692) at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:531) at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:495) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupConnection(RpcClientImpl.java:410) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.setupIOstreams(RpcClientImpl.java:716) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.writeRequest(RpcClientImpl.java:887) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.tracedWriteRequest(RpcClientImpl.java:856) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1199) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:213) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:287) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:32831) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:379) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:201) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:63) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:364) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:338) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:65) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:722) Standard Output 2017-02-23 20:59:36,967 INFO [main] hbase.HBaseTestingUtility(1003): Starting up minicluster with 1 master(s) and 3 regionserver(s) and 3 datanode(s) 2017-02-23 20:59:37,008 INFO [main] hbase.HBaseTestingUtility(471): Created new mini-cluster data directory: /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364, deleteOnExit=true 2017-02-23 20:59:37,009 INFO [main] hbase.HBaseTestingUtility(718): Setting test.cache.data to /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/cache_data in system properties and HBase conf 2017-02-23 20:59:37,010 INFO [main] hbase.HBaseTestingUtility(718): Setting hadoop.tmp.dir to /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/hadoop_tmp in system properties and HBase conf 2017-02-23 20:59:37,011 INFO [main] hbase.HBaseTestingUtility(718): Setting hadoop.log.dir to /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/hadoop_logs in system properties and HBase conf 2017-02-23 20:59:37,012 INFO [main] hbase.HBaseTestingUtility(718): Setting mapreduce.cluster.local.dir to /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/mapred_local in system properties and HBase conf 2017-02-23 20:59:37,013 INFO [main] hbase.HBaseTestingUtility(718): Setting mapreduce.cluster.temp.dir to /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/mapred_temp in system properties and HBase conf 2017-02-23 20:59:37,014 INFO [main] hbase.HBaseTestingUtility(709): read short circuit is OFF 2017-02-23 20:59:37,638 WARN [main] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2017-02-23 20:59:38,292 DEBUG [main] fs.HFileSystem(225): The file system is not a DistributedFileSystem. Skipping on block location reordering Formatting using clusterid: testClusterID 2017-02-23 20:59:39,382 WARN [main] impl.MetricsConfig(125): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2017-02-23 20:59:39,915 INFO [main] log.Slf4jLog(67): Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog 2017-02-23 20:59:39,991 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:40,028 INFO [main] log.Slf4jLog(67): Extract jar:file:/grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.3.2.6.0.0-SNAPSHOT/hadoop-hdfs-2.7.3.2.6.0.0-SNAPSHOT-tests.jar!/webapps/hdfs to /tmp/Jetty_localhost_52287_hdfs____xexpcu/webapp 2017-02-23 20:59:40,239 INFO [main] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:52287 2017-02-23 20:59:40,943 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 0, pendingReplications = 0. 2017-02-23 20:59:41,365 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:41,370 INFO [main] log.Slf4jLog(67): Extract jar:file:/grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.3.2.6.0.0-SNAPSHOT/hadoop-hdfs-2.7.3.2.6.0.0-SNAPSHOT-tests.jar!/webapps/datanode to /tmp/Jetty_localhost_55126_datanode____.rll2is/webapp 2017-02-23 20:59:41,533 INFO [main] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55126 2017-02-23 20:59:42,044 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:42,049 INFO [main] log.Slf4jLog(67): Extract jar:file:/grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.3.2.6.0.0-SNAPSHOT/hadoop-hdfs-2.7.3.2.6.0.0-SNAPSHOT-tests.jar!/webapps/datanode to /tmp/Jetty_localhost_41399_datanode____vlq09p/webapp 2017-02-23 20:59:42,399 INFO [main] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41399 2017-02-23 20:59:42,651 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:42,655 INFO [main] log.Slf4jLog(67): Extract jar:file:/grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-hdfs/2.7.3.2.6.0.0-SNAPSHOT/hadoop-hdfs-2.7.3.2.6.0.0-SNAPSHOT-tests.jar!/webapps/datanode to /tmp/Jetty_localhost_57111_datanode____.bv30bu/webapp 2017-02-23 20:59:42,772 INFO [main] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:57111 2017-02-23 20:59:43,353 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-a4b6df0b-6df2-40e4-b84e-a3e08d27b5e5 node DatanodeRegistration(127.0.0.1:40826, datanodeUuid=bb16a357-7816-4797-b3cf-3459194741a4, infoPort=36584, infoSecurePort=0, ipcPort=52469, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,354 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-1d04d79f-8774-4c2d-97fc-4f787cfea73e node DatanodeRegistration(127.0.0.1:49717, datanodeUuid=6e1c0f26-b1ae-4a7d-8951-00f0d363315e, infoPort=42196, infoSecurePort=0, ipcPort=33263, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,354 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-82808e2b-b76a-406b-9107-33cddf211457 node DatanodeRegistration(127.0.0.1:40826, datanodeUuid=bb16a357-7816-4797-b3cf-3459194741a4, infoPort=36584, infoSecurePort=0, ipcPort=52469, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,354 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-ea485d78-3d8b-4921-9603-44be19373cef node DatanodeRegistration(127.0.0.1:49717, datanodeUuid=6e1c0f26-b1ae-4a7d-8951-00f0d363315e, infoPort=42196, infoSecurePort=0, ipcPort=33263, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,404 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-6ca03d09-d28c-4cd9-b0ca-6554b2bd7067 node DatanodeRegistration(127.0.0.1:42199, datanodeUuid=5c085dc6-9635-40fb-8569-b95da28d0c99, infoPort=56443, infoSecurePort=0, ipcPort=48094, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,405 INFO [Block report processor] blockmanagement.BlockManager(1978): BLOCK* processReport: from storage DS-89452135-33a6-49e9-b6cb-15581b841ff5 node DatanodeRegistration(127.0.0.1:42199, datanodeUuid=5c085dc6-9635-40fb-8569-b95da28d0c99, infoPort=56443, infoSecurePort=0, ipcPort=48094, storageInfo=lv=-56;cid=testClusterID;nsid=1609573027;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2017-02-23 20:59:43,509 INFO [main] zookeeper.MiniZooKeeperCluster(276): Started MiniZooKeeperCluster and ran successful 'stat' on client port=52433 2017-02-23 20:59:43,523 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:43,526 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:44,096 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 0, pendingReplications = 0. 2017-02-23 20:59:44,241 INFO [main] util.FSUtils(763): Created version file at hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b with version=8 2017-02-23 20:59:44,301 DEBUG [main] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 20:59:44,471 INFO [main] client.ConnectionUtils(104): master/xxx/xxx:0 server-side HConnection retries=350 2017-02-23 20:59:44,482 INFO [main] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 20:59:44,492 INFO [main] ipc.RpcServer$Listener(608): master/xxx/xxx:0: started 10 reader(s). 2017-02-23 20:59:44,532 INFO [main] hfile.CacheConfig(530): Allocating LruBlockCache size=995.58 MB, blockSize=64 KB 2017-02-23 20:59:44,537 DEBUG [main] hfile.CacheConfig(544): Trying to use Internal l2 cache 2017-02-23 20:59:44,537 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:44,538 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:44,541 INFO [main] mob.MobFileCache(121): MobFileCache is initialized, and the cache size is 1000 2017-02-23 20:59:44,543 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:44,550 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:44,609 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=master:39475 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:44,801 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:394750x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:44,802 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): master:39475-0x15a6cc5bffa0000 connected 2017-02-23 20:59:45,094 DEBUG [main] zookeeper.ZKUtil(369): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2017-02-23 20:59:45,095 DEBUG [main] zookeeper.ZKUtil(369): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 20:59:45,099 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 20:59:45,099 INFO [RpcServer.listener,port=39475] ipc.RpcServer$Listener(739): RpcServer.listener,port=39475: starting 2017-02-23 20:59:45,100 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=39475 2017-02-23 20:59:45,101 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=39475 2017-02-23 20:59:45,101 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=39475 2017-02-23 20:59:45,102 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=39475 2017-02-23 20:59:45,102 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=39475 2017-02-23 20:59:45,102 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=39475 2017-02-23 20:59:45,103 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=39475 2017-02-23 20:59:45,103 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=39475 2017-02-23 20:59:45,103 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=39475 2017-02-23 20:59:45,104 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=39475 2017-02-23 20:59:45,104 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=39475 2017-02-23 20:59:45,105 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=39475 2017-02-23 20:59:45,105 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=39475 2017-02-23 20:59:45,106 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=39475 2017-02-23 20:59:45,106 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=39475 2017-02-23 20:59:45,106 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=39475 2017-02-23 20:59:45,107 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=39475 2017-02-23 20:59:45,107 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=39475 2017-02-23 20:59:45,108 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=39475 2017-02-23 20:59:45,108 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=39475 2017-02-23 20:59:45,109 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=39475 2017-02-23 20:59:45,109 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=39475 2017-02-23 20:59:45,109 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=39475 2017-02-23 20:59:45,110 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=39475 2017-02-23 20:59:45,110 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=39475 2017-02-23 20:59:45,111 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=39475 2017-02-23 20:59:45,111 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=39475 2017-02-23 20:59:45,112 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=39475 2017-02-23 20:59:45,112 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=39475 2017-02-23 20:59:45,118 INFO [main] master.HMaster(429): hbase.rootdir=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b, hbase.cluster.distributed=false 2017-02-23 20:59:45,127 DEBUG [main] impl.BackupManager(136): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2017-02-23 20:59:45,127 DEBUG [main] impl.BackupManager(137): Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager 2017-02-23 20:59:45,127 DEBUG [main] impl.BackupManager(138): Added master observer: org.apache.hadoop.hbase.backup.master.BackupController 2017-02-23 20:59:45,136 INFO [main] master.HMaster(1897): Adding backup master ZNode /hbase/backup-masters/xxx,39475,1487883584518 2017-02-23 20:59:45,176 DEBUG [main] zookeeper.ZKUtil(367): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/xxx,39475,1487883584518 2017-02-23 20:59:45,200 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2017-02-23 20:59:45,202 DEBUG [xxx:39475.activeMasterManager] zookeeper.ZKUtil(367): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 20:59:45,203 INFO [xxx:39475.activeMasterManager] master.ActiveMasterManager(170): Deleting ZNode for /hbase/backup-masters/xxx,39475,1487883584518 from backup master directory 2017-02-23 20:59:45,206 DEBUG [main-EventThread] zookeeper.ZKUtil(367): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 20:59:45,206 DEBUG [main-EventThread] master.ActiveMasterManager(126): A master is now available 2017-02-23 20:59:45,229 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): master:39475-0x15a6cc5bffa0000, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/xxx,39475,1487883584518 2017-02-23 20:59:45,230 WARN [xxx:39475.activeMasterManager] hbase.ZNodeClearer(58): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2017-02-23 20:59:45,231 INFO [xxx:39475.activeMasterManager] master.ActiveMasterManager(179): Registered Active Master=xxx,39475,1487883584518 2017-02-23 20:59:45,233 DEBUG [main] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 20:59:45,234 INFO [main] client.ConnectionUtils(104): regionserver/xxx/xxx:0 server-side HConnection retries=350 2017-02-23 20:59:45,235 INFO [main] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 20:59:45,240 INFO [main] ipc.RpcServer$Listener(608): regionserver/xxx/xxx:0: started 10 reader(s). 2017-02-23 20:59:45,250 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,251 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,254 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,258 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,260 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=regionserver:36349 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,271 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:363490x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,272 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): regionserver:36349-0x15a6cc5bffa0001 connected 2017-02-23 20:59:45,272 DEBUG [main] zookeeper.ZKUtil(367): regionserver:36349-0x15a6cc5bffa0001, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 20:59:45,274 DEBUG [main] zookeeper.ZKUtil(369): regionserver:36349-0x15a6cc5bffa0001, quorum=localhost:52433, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 20:59:45,274 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 20:59:45,274 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=36349 2017-02-23 20:59:45,274 INFO [RpcServer.listener,port=36349] ipc.RpcServer$Listener(739): RpcServer.listener,port=36349: starting 2017-02-23 20:59:45,275 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=36349 2017-02-23 20:59:45,276 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=36349 2017-02-23 20:59:45,276 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=36349 2017-02-23 20:59:45,276 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=36349 2017-02-23 20:59:45,277 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=36349 2017-02-23 20:59:45,277 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=36349 2017-02-23 20:59:45,277 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=36349 2017-02-23 20:59:45,278 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=36349 2017-02-23 20:59:45,278 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=36349 2017-02-23 20:59:45,278 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=36349 2017-02-23 20:59:45,279 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=36349 2017-02-23 20:59:45,279 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=36349 2017-02-23 20:59:45,279 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=36349 2017-02-23 20:59:45,280 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=36349 2017-02-23 20:59:45,280 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=36349 2017-02-23 20:59:45,281 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=36349 2017-02-23 20:59:45,281 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=36349 2017-02-23 20:59:45,281 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=36349 2017-02-23 20:59:45,282 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=36349 2017-02-23 20:59:45,282 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=36349 2017-02-23 20:59:45,282 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=36349 2017-02-23 20:59:45,283 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=36349 2017-02-23 20:59:45,283 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=36349 2017-02-23 20:59:45,283 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=36349 2017-02-23 20:59:45,284 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=36349 2017-02-23 20:59:45,284 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=36349 2017-02-23 20:59:45,284 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=36349 2017-02-23 20:59:45,285 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=36349 2017-02-23 20:59:45,292 INFO [main] http.HttpRequestLog(69): Http request log for http.requests.regionserver is not defined 2017-02-23 20:59:45,294 INFO [main] http.HttpServer(852): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2017-02-23 20:59:45,296 INFO [main] http.HttpServer(830): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2017-02-23 20:59:45,296 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2017-02-23 20:59:45,296 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2017-02-23 20:59:45,300 INFO [main] http.HttpServer(1083): Jetty bound to port 59113 2017-02-23 20:59:45,301 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:45,360 DEBUG [xxx:39475.activeMasterManager] util.FSUtils(915): Created cluster ID file at hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/hbase.id with ID: 20112447-518e-4947-b590-056bd440301e 2017-02-23 20:59:45,363 INFO [main] log.Slf4jLog(67): Started SelectChannelConnector@0.0.0.0:59113 2017-02-23 20:59:45,398 INFO [xxx:39475.activeMasterManager] master.MasterFileSystem(560): BOOTSTRAP: creating hbase:meta region 2017-02-23 20:59:45,400 INFO [xxx:39475.activeMasterManager] regionserver.HRegion(6360): creating HRegion hbase:meta HTD == 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|'}, {NAME => 'info', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '10', TTL => 'FOREVER', MIN_VERSIONS => '0', CACHE_DATA_IN_L1 => 'true', KEEP_DELETED_CELLS => 'FALSE', BLOCKSIZE => '8192', IN_MEMORY => 'false', BLOCKCACHE => 'false'} RootDir = hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b Table name == hbase:meta 2017-02-23 20:59:45,411 DEBUG [main] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 20:59:45,412 INFO [main] client.ConnectionUtils(104): regionserver/xxx/xxx:0 server-side HConnection retries=350 2017-02-23 20:59:45,413 INFO [main] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 20:59:45,416 INFO [main] ipc.RpcServer$Listener(608): regionserver/xxx/xxx:0: started 10 reader(s). 2017-02-23 20:59:45,417 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,417 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,421 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,426 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,430 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=regionserver:40055 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,446 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:400550x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,447 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): regionserver:40055-0x15a6cc5bffa0002 connected 2017-02-23 20:59:45,447 DEBUG [main] zookeeper.ZKUtil(367): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 20:59:45,449 DEBUG [main] zookeeper.ZKUtil(369): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 20:59:45,449 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 20:59:45,450 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=40055 2017-02-23 20:59:45,450 INFO [RpcServer.listener,port=40055] ipc.RpcServer$Listener(739): RpcServer.listener,port=40055: starting 2017-02-23 20:59:45,450 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=40055 2017-02-23 20:59:45,451 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=40055 2017-02-23 20:59:45,451 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=40055 2017-02-23 20:59:45,451 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=40055 2017-02-23 20:59:45,452 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=40055 2017-02-23 20:59:45,452 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=40055 2017-02-23 20:59:45,453 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=40055 2017-02-23 20:59:45,453 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=40055 2017-02-23 20:59:45,453 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=40055 2017-02-23 20:59:45,454 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=40055 2017-02-23 20:59:45,454 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=40055 2017-02-23 20:59:45,454 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=40055 2017-02-23 20:59:45,455 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=40055 2017-02-23 20:59:45,455 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=40055 2017-02-23 20:59:45,455 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=40055 2017-02-23 20:59:45,456 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=40055 2017-02-23 20:59:45,456 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=40055 2017-02-23 20:59:45,456 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=40055 2017-02-23 20:59:45,457 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=40055 2017-02-23 20:59:45,457 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=40055 2017-02-23 20:59:45,458 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=40055 2017-02-23 20:59:45,458 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=40055 2017-02-23 20:59:45,459 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=40055 2017-02-23 20:59:45,459 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=40055 2017-02-23 20:59:45,459 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=40055 2017-02-23 20:59:45,460 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=40055 2017-02-23 20:59:45,460 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=40055 2017-02-23 20:59:45,460 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=40055 2017-02-23 20:59:45,461 INFO [main] http.HttpRequestLog(69): Http request log for http.requests.regionserver is not defined 2017-02-23 20:59:45,462 INFO [main] http.HttpServer(852): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2017-02-23 20:59:45,463 INFO [xxx:39475.activeMasterManager] wal.WALFactory(145): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.DefaultWALProvider 2017-02-23 20:59:45,463 INFO [main] http.HttpServer(830): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2017-02-23 20:59:45,463 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2017-02-23 20:59:45,464 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2017-02-23 20:59:45,465 INFO [main] http.HttpServer(1083): Jetty bound to port 43168 2017-02-23 20:59:45,465 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:45,486 INFO [main] log.Slf4jLog(67): Started SelectChannelConnector@0.0.0.0:43168 2017-02-23 20:59:45,491 INFO [xxx:39475.activeMasterManager] wal.FSHLog(581): WAL configuration: blocksize=128 MB, rollsize=121.60 MB, prefix=hregion-94346745.default, suffix=, logDir=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/WALs/hregion-94346745, archiveDir=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/oldWALs 2017-02-23 20:59:45,515 DEBUG [main] impl.BackupManager(160): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager 2017-02-23 20:59:45,517 INFO [main] client.ConnectionUtils(104): regionserver/xxx/xxx:0 server-side HConnection retries=350 2017-02-23 20:59:45,517 INFO [main] ipc.SimpleRpcScheduler(196): Using fifo as user call queue, count=1 2017-02-23 20:59:45,523 INFO [main] ipc.RpcServer$Listener(608): regionserver/xxx/xxx:0: started 10 reader(s). 2017-02-23 20:59:45,524 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,524 INFO [main] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,527 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,528 INFO [xxx:39475.activeMasterManager] wal.FSHLog(1620): Slow sync cost: 19 ms, current pipeline: [] 2017-02-23 20:59:45,528 INFO [xxx:39475.activeMasterManager] wal.FSHLog(1048): New WAL /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/WALs/hregion-94346745/hregion-94346745.default.1487883585491 2017-02-23 20:59:45,530 INFO [main] fs.HFileSystem(256): Added intercepting call to namenode#getBlockLocations so can do block reordering using class class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2017-02-23 20:59:45,532 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=regionserver:60846 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,553 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:608460x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,553 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): regionserver:60846-0x15a6cc5bffa0003 connected 2017-02-23 20:59:45,554 DEBUG [main] zookeeper.ZKUtil(367): regionserver:60846-0x15a6cc5bffa0003, quorum=localhost:52433, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2017-02-23 20:59:45,556 DEBUG [main] zookeeper.ZKUtil(369): regionserver:60846-0x15a6cc5bffa0003, quorum=localhost:52433, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2017-02-23 20:59:45,557 INFO [RpcServer.responder] ipc.RpcServer$Responder(927): RpcServer.responder: starting 2017-02-23 20:59:45,557 INFO [RpcServer.listener,port=60846] ipc.RpcServer$Listener(739): RpcServer.listener,port=60846: starting 2017-02-23 20:59:45,557 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=0,queue=0,port=60846 2017-02-23 20:59:45,558 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=1,queue=0,port=60846 2017-02-23 20:59:45,558 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=2,queue=0,port=60846 2017-02-23 20:59:45,558 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=3,queue=0,port=60846 2017-02-23 20:59:45,559 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=4,queue=0,port=60846 2017-02-23 20:59:45,559 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.default.handler=5,queue=0,port=60846 2017-02-23 20:59:45,559 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=0,queue=0,port=60846 2017-02-23 20:59:45,560 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=1,queue=1,port=60846 2017-02-23 20:59:45,560 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=2,queue=0,port=60846 2017-02-23 20:59:45,561 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=3,queue=1,port=60846 2017-02-23 20:59:45,561 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=4,queue=0,port=60846 2017-02-23 20:59:45,561 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=5,queue=1,port=60846 2017-02-23 20:59:45,562 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=6,queue=0,port=60846 2017-02-23 20:59:45,562 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=7,queue=1,port=60846 2017-02-23 20:59:45,562 DEBUG [xxx:39475.activeMasterManager] regionserver.HRegion(797): Instantiated hbase:meta,,1.1588230740 2017-02-23 20:59:45,562 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=8,queue=0,port=60846 2017-02-23 20:59:45,563 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=9,queue=1,port=60846 2017-02-23 20:59:45,563 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=10,queue=0,port=60846 2017-02-23 20:59:45,563 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=11,queue=1,port=60846 2017-02-23 20:59:45,564 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=12,queue=0,port=60846 2017-02-23 20:59:45,564 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=13,queue=1,port=60846 2017-02-23 20:59:45,564 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=14,queue=0,port=60846 2017-02-23 20:59:45,564 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=15,queue=1,port=60846 2017-02-23 20:59:45,565 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=16,queue=0,port=60846 2017-02-23 20:59:45,565 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=17,queue=1,port=60846 2017-02-23 20:59:45,565 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=18,queue=0,port=60846 2017-02-23 20:59:45,566 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.priority.handler=19,queue=1,port=60846 2017-02-23 20:59:45,566 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=0,queue=0,port=60846 2017-02-23 20:59:45,566 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=1,queue=0,port=60846 2017-02-23 20:59:45,566 DEBUG [main] ipc.RpcExecutor(129): Started RpcServer.FifoWFPBQ.replication.handler=2,queue=0,port=60846 2017-02-23 20:59:45,567 INFO [main] http.HttpRequestLog(69): Http request log for http.requests.regionserver is not defined 2017-02-23 20:59:45,568 INFO [main] http.HttpServer(852): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2017-02-23 20:59:45,568 INFO [main] http.HttpServer(830): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2017-02-23 20:59:45,569 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2017-02-23 20:59:45,569 INFO [main] http.HttpServer(837): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2017-02-23 20:59:45,570 INFO [main] http.HttpServer(1083): Jetty bound to port 42045 2017-02-23 20:59:45,570 INFO [main] log.Slf4jLog(67): jetty-6.1.26.hwx 2017-02-23 20:59:45,585 INFO [main] log.Slf4jLog(67): Started SelectChannelConnector@0.0.0.0:42045 2017-02-23 20:59:45,616 INFO [StoreOpener-1588230740-1] hfile.CacheConfig(278): blockCache=LruBlockCache{blockCount=0, currentSize=1071760, freeSize=1042864304, maxSize=1043936064, heapSize=1071760, minSize=991739264, minFactor=0.95, multiSize=495869632, multiFactor=0.5, singleSize=247934816, singleFactor=0.25}, cacheDataOnRead=false, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-02-23 20:59:45,626 INFO [StoreOpener-1588230740-1] compactions.CompactionConfiguration(133): size [134217728, 9223372036854775807); files [3, 10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, base window in milliseconds 21600000, windows per tier 4,incoming window min 6 2017-02-23 20:59:45,633 DEBUG [StoreOpener-1588230740-1] regionserver.HRegionFileSystem(202): No StoreFiles for: hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/hbase/meta/1588230740/info 2017-02-23 20:59:45,638 DEBUG [StoreOpener-1588230740-1] util.ChecksumType$2(70): org.apache.hadoop.util.PureJavaCrc32 available 2017-02-23 20:59:45,638 DEBUG [StoreOpener-1588230740-1] util.ChecksumType$3(113): org.apache.hadoop.util.PureJavaCrc32C available 2017-02-23 20:59:45,643 INFO [M:0;xxx:39475] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x43c2c38b connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,643 INFO [RS:0;xxx:36349] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x5c26d61b connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,643 INFO [RS:2;xxx:60846] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x79b2b5e0 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,643 INFO [RS:1;xxx:40055] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x66090cca connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 20:59:45,652 DEBUG [xxx:39475.activeMasterManager] regionserver.HRegion(4092): Found 0 recovered edits file(s) under hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/hbase/meta/1588230740 2017-02-23 20:59:45,681 DEBUG [xxx:39475.activeMasterManager] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/hbase/meta/1588230740/recovered.edits/2.seqid to file, newSeqId=2, maxSeqId=0 2017-02-23 20:59:45,681 INFO [xxx:39475.activeMasterManager] regionserver.HRegion(924): Onlined 1588230740; next sequenceid=2 2017-02-23 20:59:45,682 DEBUG [xxx:39475.activeMasterManager] regionserver.HRegion(1453): Closing hbase:meta,,1.1588230740: disabling compactions & flushes 2017-02-23 20:59:45,682 DEBUG [xxx:39475.activeMasterManager] regionserver.HRegion(1480): Updates disabled for region hbase:meta,,1.1588230740 2017-02-23 20:59:45,683 INFO [StoreCloserThread-hbase:meta,,1.1588230740-1] regionserver.HStore(917): Closed info 2017-02-23 20:59:45,683 INFO [xxx:39475.activeMasterManager] regionserver.HRegion(1581): Closed hbase:meta,,1.1588230740 2017-02-23 20:59:45,684 DEBUG [xxx:39475.activeMasterManager] wal.FSHLog(1245): Closing WAL writer in /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/WALs/hregion-94346745 2017-02-23 20:59:45,688 DEBUG [M:0;xxx:39475-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x43c2c38b0x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,688 DEBUG [M:0;xxx:39475-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x43c2c38b-0x15a6cc5bffa0004 connected 2017-02-23 20:59:45,708 DEBUG [xxx:39475.activeMasterManager] wal.FSHLog(1203): Moved 1 WAL file(s) to /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/oldWALs 2017-02-23 20:59:45,709 INFO [xxx:39475.activeMasterManager] wal.FSHLog(1206): Closed WAL: FSHLog hregion-94346745.default:(num 1487883585491) 2017-02-23 20:59:45,722 DEBUG [RS:0;xxx:36349-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x5c26d61b0x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,722 DEBUG [RS:0;xxx:36349-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x5c26d61b-0x15a6cc5bffa0005 connected 2017-02-23 20:59:45,722 DEBUG [RS:1;xxx:40055-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x66090cca0x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,723 DEBUG [RS:2;xxx:60846-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x79b2b5e00x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 20:59:45,723 INFO [RS:0;xxx:36349] client.ZooKeeperRegistry(107): ClusterId read in ZooKeeper is null 2017-02-23 20:59:45,724 INFO [RS:2;xxx ...[truncated 1549856 bytes]... SplitLogWorker-xxx:60846] regionserver.SplitLogWorker(155): SplitLogWorker xxx,60846,1487883585524 exiting 2017-02-23 21:38:45,104 INFO [SplitLogWorker-xxx:40055] regionserver.SplitLogWorker(146): SplitLogWorker interrupted. Exiting. 2017-02-23 21:38:45,106 INFO [SplitLogWorker-xxx:40055] regionserver.SplitLogWorker(155): SplitLogWorker xxx,40055,1487883585417 exiting 2017-02-23 21:38:45,104 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1016): Stopping infoServer 2017-02-23 21:38:45,108 INFO [RS:2;xxx:60846] log.Slf4jLog(67): Stopped SelectChannelConnector@0.0.0.0:0 2017-02-23 21:38:45,127 INFO [RS:1;xxx:40055] log.Slf4jLog(67): Stopped SelectChannelConnector@0.0.0.0:0 2017-02-23 21:38:45,193 INFO [regionserver/xxx/xxx:0.leaseChecker] regionserver.Leases(146): regionserver/xxx/xxx:0.leaseChecker closing leases 2017-02-23 21:38:45,193 INFO [regionserver/xxx/xxx:0.leaseChecker] regionserver.Leases(149): regionserver/xxx/xxx:0.leaseChecker closed leases 2017-02-23 21:38:45,210 INFO [RS:2;xxx:60846] regionserver.HeapMemoryManager(199): Stoping HeapMemoryTuner chore. 2017-02-23 21:38:45,210 INFO [MemStoreFlusher.0] regionserver.MemStoreFlusher$FlushHandler(280): MemStoreFlusher.0 exiting 2017-02-23 21:38:45,212 INFO [MemStoreFlusher.1] regionserver.MemStoreFlusher$FlushHandler(280): MemStoreFlusher.1 exiting 2017-02-23 21:38:45,210 INFO [RS:2;xxx:60846] regionserver.LogRollRegionServerProcedureManager(96): Stopping RegionServerBackupManager gracefully. 2017-02-23 21:38:45,212 INFO [RS:2;xxx:60846] snapshot.RegionServerSnapshotManager(136): Stopping RegionServerSnapshotManager gracefully. 2017-02-23 21:38:45,213 INFO [RS:2;xxx:60846] flush.RegionServerFlushTableProcedureManager(115): Stopping region server flush procedure manager gracefully. 2017-02-23 21:38:45,214 DEBUG [RS_CLOSE_REGION-xxx:60846-1] handler.CloseRegionHandler(119): Processing close of hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d. 2017-02-23 21:38:45,214 DEBUG [RS_CLOSE_REGION-xxx:60846-2] handler.CloseRegionHandler(119): Processing close of scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7. 2017-02-23 21:38:45,214 INFO [RS:2;xxx:60846] regionserver.HRegionServer(1066): stopping server xxx,60846,1487883585524 2017-02-23 21:38:45,214 DEBUG [RS:2;xxx:60846] zookeeper.MetaTableLocator(627): Stopping MetaTableLocator 2017-02-23 21:38:45,215 INFO [RS:2;xxx:60846] client.ConnectionManager$HConnectionImplementation(1687): Closing zookeeper sessionid=0x15a6cc5bffa0007 2017-02-23 21:38:45,215 DEBUG [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1453): Closing scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7.: disabling compactions & flushes 2017-02-23 21:38:45,214 DEBUG [RS_CLOSE_REGION-xxx:60846-0] handler.CloseRegionHandler(119): Processing close of scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116. 2017-02-23 21:38:45,216 DEBUG [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1480): Updates disabled for region scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7. 2017-02-23 21:38:45,215 DEBUG [RS_CLOSE_REGION-xxx:60846-1] regionserver.HRegion(1453): Closing hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d.: disabling compactions & flushes 2017-02-23 21:38:45,217 DEBUG [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegion(1453): Closing scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116.: disabling compactions & flushes 2017-02-23 21:38:45,217 DEBUG [RS_CLOSE_REGION-xxx:60846-1] regionserver.HRegion(1480): Updates disabled for region hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d. 2017-02-23 21:38:45,217 DEBUG [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegion(1480): Updates disabled for region scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116. 2017-02-23 21:38:45,217 INFO [StoreCloserThread-scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,218 INFO [StoreCloserThread-hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d.-1] regionserver.HStore(917): Closed meta 2017-02-23 21:38:45,218 INFO [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegion(2275): Started memstore flush for scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116., current region memstore size 1.48 MB, and 1/1 column families' memstores are being flushed. 2017-02-23 21:38:45,218 INFO [StoreCloserThread-hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d.-1] regionserver.HStore(917): Closed session 2017-02-23 21:38:45,218 WARN [RS_CLOSE_REGION-xxx:60846-0] wal.FSHLog(1798): Couldn't find oldest seqNum for the region we are about to flush: [2ebb0d5919d07b86dd1974b31c284116] 2017-02-23 21:38:45,235 INFO [RS:1;xxx:40055] regionserver.HeapMemoryManager(199): Stoping HeapMemoryTuner chore. 2017-02-23 21:38:45,235 DEBUG [RS:2;xxx:60846] ipc.RpcClientImpl(1115): Stopping rpc client 2017-02-23 21:38:45,244 INFO [MemStoreFlusher.1] regionserver.MemStoreFlusher$FlushHandler(280): MemStoreFlusher.1 exiting 2017-02-23 21:38:45,235 INFO [MemStoreFlusher.0] regionserver.MemStoreFlusher$FlushHandler(280): MemStoreFlusher.0 exiting 2017-02-23 21:38:45,245 INFO [RS:1;xxx:40055] regionserver.LogRollRegionServerProcedureManager(96): Stopping RegionServerBackupManager gracefully. 2017-02-23 21:38:45,245 INFO [RS:2;xxx:60846] regionserver.HRegionServer(1355): Waiting on 4 regions to close 2017-02-23 21:38:45,245 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(1359): {2ebb0d5919d07b86dd1974b31c284116=scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116., 707d7836207e13c20979ed59832db12d=hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d., 3daf14ab71a9b2da8fadb0aa070bc0c7=scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7., c42134aa6d3a66b14c9de0a02e7833ff=scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff.} 2017-02-23 21:38:45,245 INFO [RS:1;xxx:40055] snapshot.RegionServerSnapshotManager(136): Stopping RegionServerSnapshotManager gracefully. 2017-02-23 21:38:45,245 INFO [RS:1;xxx:40055] flush.RegionServerFlushTableProcedureManager(115): Stopping region server flush procedure manager gracefully. 2017-02-23 21:38:45,246 DEBUG [RS_CLOSE_REGION-xxx:40055-0] handler.CloseRegionHandler(119): Processing close of scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501. 2017-02-23 21:38:45,274 DEBUG [RS_CLOSE_REGION-xxx:40055-1] handler.CloseRegionHandler(119): Processing close of scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a. 2017-02-23 21:38:45,274 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1066): stopping server xxx,40055,1487883585417 2017-02-23 21:38:45,283 DEBUG [RS:1;xxx:40055] zookeeper.MetaTableLocator(627): Stopping MetaTableLocator 2017-02-23 21:38:45,274 DEBUG [RS_CLOSE_REGION-xxx:40055-2] handler.CloseRegionHandler(119): Processing close of scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813. 2017-02-23 21:38:45,283 INFO [RS:1;xxx:40055] client.ConnectionManager$HConnectionImplementation(1687): Closing zookeeper sessionid=0x15a6cc5bffa0006 2017-02-23 21:38:45,283 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1453): Closing scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a.: disabling compactions & flushes 2017-02-23 21:38:45,279 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1453): Closing scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501.: disabling compactions & flushes 2017-02-23 21:38:45,291 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1480): Updates disabled for region scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a. 2017-02-23 21:38:45,291 INFO [regionserver/xxx/xxx:0.leaseChecker] regionserver.Leases(146): regionserver/xxx/xxx:0.leaseChecker closing leases 2017-02-23 21:38:45,291 INFO [regionserver/xxx/xxx:0.leaseChecker] regionserver.Leases(149): regionserver/xxx/xxx:0.leaseChecker closed leases 2017-02-23 21:38:45,290 DEBUG [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegion(1453): Closing scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813.: disabling compactions & flushes 2017-02-23 21:38:45,291 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(2275): Started memstore flush for scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a., current region memstore size 176 B, and 1/1 column families' memstores are being flushed. 2017-02-23 21:38:45,291 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1480): Updates disabled for region scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501. 2017-02-23 21:38:45,292 WARN [RS_CLOSE_REGION-xxx:40055-1] wal.FSHLog(1798): Couldn't find oldest seqNum for the region we are about to flush: [23fc969c7071ae33139ddeb5bc8ee15a] 2017-02-23 21:38:45,292 DEBUG [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegion(1480): Updates disabled for region scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813. 2017-02-23 21:38:45,292 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(2275): Started memstore flush for scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501., current region memstore size 1.48 MB, and 1/1 column families' memstores are being flushed. 2017-02-23 21:38:45,293 INFO [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegion(2275): Started memstore flush for scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813., current region memstore size 1.47 MB, and 1/1 column families' memstores are being flushed. 2017-02-23 21:38:45,293 WARN [RS_CLOSE_REGION-xxx:40055-0] wal.FSHLog(1798): Couldn't find oldest seqNum for the region we are about to flush: [dcaf5c835f54a435eca58220bd0a5501] 2017-02-23 21:38:45,293 WARN [RS_CLOSE_REGION-xxx:40055-2] wal.FSHLog(1798): Couldn't find oldest seqNum for the region we are about to flush: [d1069f68b5c7116c9922d552b76db813] 2017-02-23 21:38:45,336 DEBUG [RS:1;xxx:40055] ipc.RpcClientImpl(1115): Stopping rpc client 2017-02-23 21:38:45,337 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1355): Waiting on 5 regions to close 2017-02-23 21:38:45,337 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(1359): {dcaf5c835f54a435eca58220bd0a5501=scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501., 23fc969c7071ae33139ddeb5bc8ee15a=scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a., d1069f68b5c7116c9922d552b76db813=scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813., 8485ddbf720b615a3fa2fa8c8bd601a4=scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4., 26b66c39851c9c347c1d0c581c409c42=scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42.} 2017-02-23 21:38:45,344 DEBUG [RS_CLOSE_REGION-xxx:60846-2] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/3daf14ab71a9b2da8fadb0aa070bc0c7/recovered.edits/4.seqid to file, newSeqId=4, maxSeqId=2 2017-02-23 21:38:45,345 DEBUG [RS_CLOSE_REGION-xxx:60846-1] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/hbase/backup/707d7836207e13c20979ed59832db12d/recovered.edits/4.seqid to file, newSeqId=4, maxSeqId=2 2017-02-23 21:38:45,363 INFO [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1581): Closed scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7. 2017-02-23 21:38:45,363 DEBUG [RS_CLOSE_REGION-xxx:60846-2] handler.CloseRegionHandler(164): Closed scantest0,zzzzz,1487883591799.3daf14ab71a9b2da8fadb0aa070bc0c7. 2017-02-23 21:38:45,363 DEBUG [RS_CLOSE_REGION-xxx:60846-2] handler.CloseRegionHandler(119): Processing close of scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff. 2017-02-23 21:38:45,363 INFO [RS_CLOSE_REGION-xxx:60846-1] regionserver.HRegion(1581): Closed hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d. 2017-02-23 21:38:45,364 DEBUG [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1453): Closing scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff.: disabling compactions & flushes 2017-02-23 21:38:45,364 DEBUG [RS_CLOSE_REGION-xxx:60846-1] handler.CloseRegionHandler(164): Closed hbase:backup,,1487883591611.707d7836207e13c20979ed59832db12d. 2017-02-23 21:38:45,365 DEBUG [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1480): Updates disabled for region scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff. 2017-02-23 21:38:45,366 INFO [StoreCloserThread-scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,427 DEBUG [RS_CLOSE_REGION-xxx:60846-2] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/c42134aa6d3a66b14c9de0a02e7833ff/recovered.edits/4.seqid to file, newSeqId=4, maxSeqId=2 2017-02-23 21:38:45,429 INFO [RS_CLOSE_REGION-xxx:60846-2] regionserver.HRegion(1581): Closed scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff. 2017-02-23 21:38:45,429 DEBUG [RS_CLOSE_REGION-xxx:60846-2] handler.CloseRegionHandler(164): Closed scantest2,zzzzz,1487883600488.c42134aa6d3a66b14c9de0a02e7833ff. 2017-02-23 21:38:45,560 INFO [xxx,40055,1487883585417_ChoreService_1] hbase.ScheduledChore(179): Chore: xxx,40055,1487883585417-MemstoreFlusherChore was stopped 2017-02-23 21:38:45,560 INFO [xxx,60846,1487883585524_ChoreService_2] hbase.ScheduledChore(179): Chore: xxx,60846,1487883585524-MemstoreFlusherChore was stopped 2017-02-23 21:38:45,603 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.DefaultStoreFlusher(87): Flushed, sequenceid=6, memsize=176, hasBloomFilter=false, into tmp file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/23fc969c7071ae33139ddeb5bc8ee15a/.tmp/7cc510ba59544352b78ce47a0ced43d7 2017-02-23 21:38:45,640 INFO [RS_CLOSE_REGION-xxx:40055-2] regionserver.DefaultStoreFlusher(87): Flushed, sequenceid=8, memsize=1.5 M, hasBloomFilter=false, into tmp file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/d1069f68b5c7116c9922d552b76db813/.tmp/482c7d6ea49d4e19aafb2a3bdc6ac372 2017-02-23 21:38:45,641 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.DefaultStoreFlusher(87): Flushed, sequenceid=10, memsize=1.5 M, hasBloomFilter=false, into tmp file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/dcaf5c835f54a435eca58220bd0a5501/.tmp/acb1538bf935435bb692fea951f4edd5 2017-02-23 21:38:45,645 INFO [RS_CLOSE_REGION-xxx:60846-0] regionserver.DefaultStoreFlusher(87): Flushed, sequenceid=10, memsize=1.5 M, hasBloomFilter=false, into tmp file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/2ebb0d5919d07b86dd1974b31c284116/.tmp/b1e2fb7b1f6f40b0a221749799fb2c24 2017-02-23 21:38:45,651 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegionFileSystem(384): Committing store file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/23fc969c7071ae33139ddeb5bc8ee15a/.tmp/7cc510ba59544352b78ce47a0ced43d7 as hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/23fc969c7071ae33139ddeb5bc8ee15a/contents/7cc510ba59544352b78ce47a0ced43d7 2017-02-23 21:38:45,656 DEBUG [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegionFileSystem(384): Committing store file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/d1069f68b5c7116c9922d552b76db813/.tmp/482c7d6ea49d4e19aafb2a3bdc6ac372 as hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/d1069f68b5c7116c9922d552b76db813/contents/482c7d6ea49d4e19aafb2a3bdc6ac372 2017-02-23 21:38:45,658 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegionFileSystem(384): Committing store file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/dcaf5c835f54a435eca58220bd0a5501/.tmp/acb1538bf935435bb692fea951f4edd5 as hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/dcaf5c835f54a435eca58220bd0a5501/contents/acb1538bf935435bb692fea951f4edd5 2017-02-23 21:38:45,660 DEBUG [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegionFileSystem(384): Committing store file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/2ebb0d5919d07b86dd1974b31c284116/.tmp/b1e2fb7b1f6f40b0a221749799fb2c24 as hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/2ebb0d5919d07b86dd1974b31c284116/contents/b1e2fb7b1f6f40b0a221749799fb2c24 2017-02-23 21:38:45,662 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.HStore(1011): Added hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/23fc969c7071ae33139ddeb5bc8ee15a/contents/7cc510ba59544352b78ce47a0ced43d7, entries=1, sequenceid=6, filesize=4.6 K 2017-02-23 21:38:45,668 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(2578): Finished memstore flush of ~176 B/176, currentsize=0 B/0 for region scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a. in 377ms, sequenceid=6, compaction requested=false 2017-02-23 21:38:45,668 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.MetricsTableSourceImpl(83): Creating new MetricsTableSourceImpl for table 2017-02-23 21:38:45,675 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HStore(1011): Added hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/dcaf5c835f54a435eca58220bd0a5501/contents/acb1538bf935435bb692fea951f4edd5, entries=8788, sequenceid=10, filesize=374.0 K 2017-02-23 21:38:45,676 INFO [RS_CLOSE_REGION-xxx:40055-2] regionserver.HStore(1011): Added hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/d1069f68b5c7116c9922d552b76db813/contents/482c7d6ea49d4e19aafb2a3bdc6ac372, entries=8787, sequenceid=8, filesize=373.9 K 2017-02-23 21:38:45,680 INFO [StoreCloserThread-scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,681 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(2578): Finished memstore flush of ~1.48 MB/1546688, currentsize=0 B/0 for region scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501. in 389ms, sequenceid=10, compaction requested=false 2017-02-23 21:38:45,682 INFO [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegion(2578): Finished memstore flush of ~1.47 MB/1546512, currentsize=0 B/0 for region scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813. in 388ms, sequenceid=8, compaction requested=false 2017-02-23 21:38:45,682 DEBUG [RS_CLOSE_REGION-xxx:40055-2] regionserver.MetricsTableSourceImpl(83): Creating new MetricsTableSourceImpl for table 2017-02-23 21:38:45,682 INFO [RS_CLOSE_REGION-xxx:60846-0] regionserver.HStore(1011): Added hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/2ebb0d5919d07b86dd1974b31c284116/contents/b1e2fb7b1f6f40b0a221749799fb2c24, entries=8788, sequenceid=10, filesize=374.0 K 2017-02-23 21:38:45,692 INFO [StoreCloserThread-scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,693 INFO [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegion(2578): Finished memstore flush of ~1.48 MB/1546688, currentsize=0 B/0 for region scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116. in 476ms, sequenceid=10, compaction requested=false 2017-02-23 21:38:45,693 DEBUG [RS_CLOSE_REGION-xxx:60846-0] regionserver.MetricsTableSourceImpl(83): Creating new MetricsTableSourceImpl for table 2017-02-23 21:38:45,695 INFO [StoreCloserThread-scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,697 INFO [StoreCloserThread-scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,699 DEBUG [RS_CLOSE_REGION-xxx:40055-1] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/23fc969c7071ae33139ddeb5bc8ee15a/recovered.edits/9.seqid to file, newSeqId=9, maxSeqId=2 2017-02-23 21:38:45,701 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1581): Closed scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a. 2017-02-23 21:38:45,701 DEBUG [RS_CLOSE_REGION-xxx:40055-1] handler.CloseRegionHandler(164): Closed scantest2,,1487883600488.23fc969c7071ae33139ddeb5bc8ee15a. 2017-02-23 21:38:45,701 DEBUG [RS_CLOSE_REGION-xxx:40055-1] handler.CloseRegionHandler(119): Processing close of scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4. 2017-02-23 21:38:45,702 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1453): Closing scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4.: disabling compactions & flushes 2017-02-23 21:38:45,702 DEBUG [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1480): Updates disabled for region scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4. 2017-02-23 21:38:45,702 DEBUG [RS_CLOSE_REGION-xxx:40055-0] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest2/dcaf5c835f54a435eca58220bd0a5501/recovered.edits/13.seqid to file, newSeqId=13, maxSeqId=2 2017-02-23 21:38:45,702 INFO [StoreCloserThread-scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,703 DEBUG [RS_CLOSE_REGION-xxx:40055-2] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/d1069f68b5c7116c9922d552b76db813/recovered.edits/11.seqid to file, newSeqId=11, maxSeqId=2 2017-02-23 21:38:45,704 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1581): Closed scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501. 2017-02-23 21:38:45,704 DEBUG [RS_CLOSE_REGION-xxx:40055-0] handler.CloseRegionHandler(164): Closed scantest2,m\xED\xED\xED\xED,1487883600488.dcaf5c835f54a435eca58220bd0a5501. 2017-02-23 21:38:45,704 DEBUG [RS_CLOSE_REGION-xxx:40055-0] handler.CloseRegionHandler(119): Processing close of scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42. 2017-02-23 21:38:45,705 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1453): Closing scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42.: disabling compactions & flushes 2017-02-23 21:38:45,705 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1480): Updates disabled for region scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42. 2017-02-23 21:38:45,705 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(2275): Started memstore flush for scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42., current region memstore size 176 B, and 1/1 column families' memstores are being flushed. 2017-02-23 21:38:45,705 INFO [RS_CLOSE_REGION-xxx:40055-2] regionserver.HRegion(1581): Closed scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813. 2017-02-23 21:38:45,705 WARN [RS_CLOSE_REGION-xxx:40055-0] wal.FSHLog(1798): Couldn't find oldest seqNum for the region we are about to flush: [26b66c39851c9c347c1d0c581c409c42] 2017-02-23 21:38:45,705 DEBUG [RS_CLOSE_REGION-xxx:40055-2] handler.CloseRegionHandler(164): Closed scantest0,aaaaa,1487883591799.d1069f68b5c7116c9922d552b76db813. 2017-02-23 21:38:45,706 DEBUG [RS_CLOSE_REGION-xxx:60846-0] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/2ebb0d5919d07b86dd1974b31c284116/recovered.edits/13.seqid to file, newSeqId=13, maxSeqId=2 2017-02-23 21:38:45,708 INFO [RS_CLOSE_REGION-xxx:60846-0] regionserver.HRegion(1581): Closed scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116. 2017-02-23 21:38:45,708 DEBUG [RS_CLOSE_REGION-xxx:60846-0] handler.CloseRegionHandler(164): Closed scantest1,m\xED\xED\xED\xED,1487883597901.2ebb0d5919d07b86dd1974b31c284116. 2017-02-23 21:38:45,711 DEBUG [RS_CLOSE_REGION-xxx:40055-1] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest1/8485ddbf720b615a3fa2fa8c8bd601a4/recovered.edits/4.seqid to file, newSeqId=4, maxSeqId=2 2017-02-23 21:38:45,713 INFO [RS_CLOSE_REGION-xxx:40055-1] regionserver.HRegion(1581): Closed scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4. 2017-02-23 21:38:45,713 DEBUG [RS_CLOSE_REGION-xxx:40055-1] handler.CloseRegionHandler(164): Closed scantest1,zzzzz,1487883597901.8485ddbf720b615a3fa2fa8c8bd601a4. 2017-02-23 21:38:45,724 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.DefaultStoreFlusher(87): Flushed, sequenceid=6, memsize=176, hasBloomFilter=false, into tmp file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/26b66c39851c9c347c1d0c581c409c42/.tmp/1e3b59b5cfa140baa1c32ddfb87740a8 2017-02-23 21:38:45,732 INFO [regionserver/xxx/xxx:0.logRoller] regionserver.LogRoller(176): LogRoller exiting. 2017-02-23 21:38:45,733 INFO [regionserver/xxx/xxx:0.logRoller] regionserver.LogRoller(176): LogRoller exiting. 2017-02-23 21:38:45,737 DEBUG [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegionFileSystem(384): Committing store file hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/26b66c39851c9c347c1d0c581c409c42/.tmp/1e3b59b5cfa140baa1c32ddfb87740a8 as hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/26b66c39851c9c347c1d0c581c409c42/contents/1e3b59b5cfa140baa1c32ddfb87740a8 2017-02-23 21:38:45,748 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HStore(1011): Added hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/26b66c39851c9c347c1d0c581c409c42/contents/1e3b59b5cfa140baa1c32ddfb87740a8, entries=1, sequenceid=6, filesize=4.6 K 2017-02-23 21:38:45,750 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(2578): Finished memstore flush of ~176 B/176, currentsize=0 B/0 for region scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42. in 45ms, sequenceid=6, compaction requested=false 2017-02-23 21:38:45,752 INFO [StoreCloserThread-scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42.-1] regionserver.HStore(917): Closed contents 2017-02-23 21:38:45,759 DEBUG [RS_CLOSE_REGION-xxx:40055-0] wal.WALSplitter(720): Wrote region seqId=hdfs://localhost:36197/user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/data/default/scantest0/26b66c39851c9c347c1d0c581c409c42/recovered.edits/9.seqid to file, newSeqId=9, maxSeqId=2 2017-02-23 21:38:45,761 INFO [RS_CLOSE_REGION-xxx:40055-0] regionserver.HRegion(1581): Closed scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42. 2017-02-23 21:38:45,761 DEBUG [RS_CLOSE_REGION-xxx:40055-0] handler.CloseRegionHandler(164): Closed scantest0,,1487883591799.26b66c39851c9c347c1d0c581c409c42. 2017-02-23 21:38:45,846 INFO [RS:2;xxx:60846] regionserver.HRegionServer(1094): stopping server xxx,60846,1487883585524; all regions closed. 2017-02-23 21:38:45,846 DEBUG [RS:2;xxx:60846] wal.FSHLog(1245): Closing WAL writer in /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/WALs/xxx,60846,1487883585524 2017-02-23 21:38:45,867 DEBUG [RS:2;xxx:60846] wal.FSHLog(1203): Moved 1 WAL file(s) to /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/oldWALs 2017-02-23 21:38:45,867 INFO [RS:2;xxx:60846] wal.FSHLog(1206): Closed WAL: FSHLog xxx%2C60846%2C1487883585524.default:(num 1487883588130) 2017-02-23 21:38:45,867 DEBUG [RS:2;xxx:60846] ipc.RpcClientImpl(1115): Stopping rpc client 2017-02-23 21:38:45,867 INFO [RS:2;xxx:60846] regionserver.Leases(146): RS:2;xxx:60846 closing leases 2017-02-23 21:38:45,867 INFO [RS:2;xxx:60846] regionserver.Leases(149): RS:2;xxx:60846 closed leases 2017-02-23 21:38:45,867 INFO [RS:2;xxx:60846] hbase.ChoreService(302): Chore service for: xxx,60846,1487883585524 had [[ScheduledChore: Name: MovedRegionsCleaner for region xxx,60846,1487883585524 Period: 120000 Unit: MILLISECONDS]] on shutdown 2017-02-23 21:38:45,868 INFO [RS:2;xxx:60846] regionserver.CompactSplitThread(413): Waiting for Split Thread to finish... 2017-02-23 21:38:45,868 INFO [RS:2;xxx:60846] regionserver.CompactSplitThread(413): Waiting for Merge Thread to finish... 2017-02-23 21:38:45,868 INFO [RS:2;xxx:60846] regionserver.CompactSplitThread(413): Waiting for Large Compaction Thread to finish... 2017-02-23 21:38:45,868 INFO [RS:2;xxx:60846] regionserver.CompactSplitThread(413): Waiting for Small Compaction Thread to finish... 2017-02-23 21:38:45,926 INFO [RS:2;xxx:60846] ipc.RpcServer(2248): Stopping server on 60846 2017-02-23 21:38:45,926 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:60846-0x15a6cc5bffa0003, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/xxx,60846,1487883585524 2017-02-23 21:38:45,927 INFO [RpcServer.listener,port=60846] ipc.RpcServer$Listener(786): RpcServer.listener,port=60846: stopping 2017-02-23 21:38:45,937 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1094): stopping server xxx,40055,1487883585417; all regions closed. 2017-02-23 21:38:45,938 DEBUG [RS:1;xxx:40055] wal.FSHLog(1245): Closing WAL writer in /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/WALs/xxx,40055,1487883585417 2017-02-23 21:38:45,982 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/xxx,60846,1487883585524 2017-02-23 21:38:45,982 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:60846-0x15a6cc5bffa0003, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/xxx,60846,1487883585524 2017-02-23 21:38:45,982 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2017-02-23 21:38:45,982 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:60846-0x15a6cc5bffa0003, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2017-02-23 21:38:45,998 INFO [RS:2;xxx:60846] regionserver.HRegionServer(1138): stopping server xxx,60846,1487883585524; zookeeper connection closed. 2017-02-23 21:38:45,999 INFO [RS:2;xxx:60846] regionserver.HRegionServer(1141): RS:2;xxx:60846 exiting 2017-02-23 21:38:45,999 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6322ffe5] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(191): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6322ffe5 2017-02-23 21:38:46,014 INFO [RpcServer.responder] ipc.RpcServer$Responder(1028): RpcServer.responder: stopped 2017-02-23 21:38:46,015 INFO [RpcServer.responder] ipc.RpcServer$Responder(931): RpcServer.responder: stopping 2017-02-23 21:38:46,388 DEBUG [RS:1;xxx:40055] wal.FSHLog(1203): Moved 1 WAL file(s) to /user/nobody/test-data/127618ec-7f4b-45df-bf8e-0d4ac353b90b/oldWALs 2017-02-23 21:38:46,389 INFO [RS:1;xxx:40055] wal.FSHLog(1206): Closed WAL: FSHLog xxx%2C40055%2C1487883585417.default:(num 1487883588130) 2017-02-23 21:38:46,389 DEBUG [RS:1;xxx:40055] ipc.RpcClientImpl(1115): Stopping rpc client 2017-02-23 21:38:46,389 INFO [RS:1;xxx:40055] regionserver.Leases(146): RS:1;xxx:40055 closing leases 2017-02-23 21:38:46,389 INFO [RS:1;xxx:40055] regionserver.Leases(149): RS:1;xxx:40055 closed leases 2017-02-23 21:38:46,390 INFO [RS:1;xxx:40055] hbase.ChoreService(302): Chore service for: xxx,40055,1487883585417 had [[ScheduledChore: Name: MovedRegionsCleaner for region xxx,40055,1487883585417 Period: 120000 Unit: MILLISECONDS]] on shutdown 2017-02-23 21:38:46,390 INFO [RS:1;xxx:40055] regionserver.CompactSplitThread(413): Waiting for Split Thread to finish... 2017-02-23 21:38:46,390 INFO [RS:1;xxx:40055] regionserver.CompactSplitThread(413): Waiting for Merge Thread to finish... 2017-02-23 21:38:46,390 INFO [RS:1;xxx:40055] regionserver.CompactSplitThread(413): Waiting for Large Compaction Thread to finish... 2017-02-23 21:38:46,390 INFO [RS:1;xxx:40055] regionserver.CompactSplitThread(413): Waiting for Small Compaction Thread to finish... 2017-02-23 21:38:46,539 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/xxx,40055,1487883585417 2017-02-23 21:38:46,638 INFO [RS:1;xxx:40055] ipc.RpcServer(2248): Stopping server on 40055 2017-02-23 21:38:46,661 INFO [RpcServer.listener,port=40055] ipc.RpcServer$Listener(786): RpcServer.listener,port=40055: stopping 2017-02-23 21:38:46,680 INFO [RpcServer.responder] ipc.RpcServer$Responder(1028): RpcServer.responder: stopped 2017-02-23 21:38:46,681 INFO [RpcServer.responder] ipc.RpcServer$Responder(931): RpcServer.responder: stopping 2017-02-23 21:38:46,758 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 0, pendingReplications = 0. 2017-02-23 21:38:46,843 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): regionserver:40055-0x15a6cc5bffa0002, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/xxx,40055,1487883585417 2017-02-23 21:38:46,889 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1138): stopping server xxx,40055,1487883585417; zookeeper connection closed. 2017-02-23 21:38:46,889 INFO [RS:1;xxx:40055] regionserver.HRegionServer(1141): RS:1;xxx:40055 exiting 2017-02-23 21:38:46,889 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a8ece25] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(191): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a8ece25 2017-02-23 21:38:46,890 INFO [main] util.JVMClusterUtil(331): Shutdown of 1 master(s) and 3 regionserver(s) complete 2017-02-23 21:38:46,958 INFO [main] zookeeper.MiniZooKeeperCluster(319): Shutdown MiniZK cluster with all ZK servers 2017-02-23 21:38:46,959 WARN [main] datanode.DirectoryScanner(380): DirectoryScanner: shutdown has been called 2017-02-23 21:38:46,991 DEBUG [xxx:39475.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(515): replicationLogCleaner-0x15a6cc5bffa0009, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Disconnected, path=null 2017-02-23 21:38:46,991 DEBUG [xxx:39475.activeMasterManager-EventThread] zookeeper.ZooKeeperWatcher(583): replicationLogCleaner-0x15a6cc5bffa0009, quorum=localhost:52433, baseZNode=/hbase Received Disconnected from ZooKeeper, ignoring 2017-02-23 21:38:47,056 INFO [main] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2017-02-23 21:38:47,190 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data5/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data6/]] heartbeating to localhost/127.0.0.1:36197] datanode.IncrementalBlockReportManager(132): IncrementalBlockReportManager interrupted 2017-02-23 21:38:47,190 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data5/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data6/]] heartbeating to localhost/127.0.0.1:36197] datanode.BPServiceActor(796): Ending block pool service for: Block pool BP-1347234630-xxx-1487883578950 (Datanode Uuid 5c085dc6-9635-40fb-8569-b95da28d0c99) service to localhost/127.0.0.1:36197 2017-02-23 21:38:47,201 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data5/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,203 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data6/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,206 WARN [main] datanode.DirectoryScanner(380): DirectoryScanner: shutdown has been called 2017-02-23 21:38:47,322 INFO [main] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2017-02-23 21:38:47,455 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data3/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data4/]] heartbeating to localhost/127.0.0.1:36197] datanode.IncrementalBlockReportManager(132): IncrementalBlockReportManager interrupted 2017-02-23 21:38:47,455 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data3/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data4/]] heartbeating to localhost/127.0.0.1:36197] datanode.BPServiceActor(796): Ending block pool service for: Block pool BP-1347234630-xxx-1487883578950 (Datanode Uuid bb16a357-7816-4797-b3cf-3459194741a4) service to localhost/127.0.0.1:36197 2017-02-23 21:38:47,457 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data3/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,457 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data4/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,460 WARN [main] datanode.DirectoryScanner(380): DirectoryScanner: shutdown has been called 2017-02-23 21:38:47,600 INFO [main] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2017-02-23 21:38:47,601 WARN [1537477200@qtp-50303574-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55126] http.HttpServer2$SelectChannelConnectorWithSafeStartup(543): HttpServer Acceptor: isRunning is false. Rechecking. 2017-02-23 21:38:47,608 WARN [1537477200@qtp-50303574-1 - Acceptor0 HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55126] http.HttpServer2$SelectChannelConnectorWithSafeStartup(552): HttpServer Acceptor: isRunning is false 2017-02-23 21:38:47,725 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data1/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data2/]] heartbeating to localhost/127.0.0.1:36197] datanode.IncrementalBlockReportManager(132): IncrementalBlockReportManager interrupted 2017-02-23 21:38:47,726 WARN [DataNode: [[[DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data1/, [DISK]file:/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data2/]] heartbeating to localhost/127.0.0.1:36197] datanode.BPServiceActor(796): Ending block pool service for: Block pool BP-1347234630-xxx-1487883578950 (Datanode Uuid 6e1c0f26-b1ae-4a7d-8951-00f0d363315e) service to localhost/127.0.0.1:36197 2017-02-23 21:38:47,727 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data1/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,733 WARN [refreshUsed-/grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/dfscluster_d8af50f3-53e4-42bf-a75f-1a5313210364/dfs/data/data2/current/BP-1347234630-xxx-1487883578950] fs.CachingGetSpaceUsed$RefreshThread(180): Thread Interrupted waiting to refresh disk information: sleep interrupted 2017-02-23 21:38:47,862 INFO [main] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2017-02-23 21:38:48,129 INFO [main] hbase.HBaseTestingUtility(1143): Minicluster is down Standard Error 2017-02-23 21:37:16,979 INFO [main] hbase.ResourceChecker(171): after: mapreduce.TestMultiTableInputFormat#testScanEmptyToEmpty Thread=762 (was 955), OpenFileDescriptor=2167 (was 2379), MaxFileDescriptor=1024000 (was 1024000), SystemLoadAverage=802 (was 2238), ProcessCount=14 (was 14), AvailableMemoryMB=91474 (was 77757) - AvailableMemoryMB LEAK? -, ConnectionCount=0 (was 0) 2017-02-23 21:37:16,979 WARN [main] hbase.ResourceChecker(134): Thread=762 is superior to 500 2017-02-23 21:37:16,980 WARN [main] hbase.ResourceChecker(134): OpenFileDescriptor=2167 is superior to 1024 2017-02-23 21:37:17,030 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:17,043 INFO [main] hbase.ResourceChecker(147): before: mapreduce.TestMultiTableInputFormat#testScanOBBToOPP Thread=762, OpenFileDescriptor=2167, MaxFileDescriptor=1024000, SystemLoadAverage=802, ProcessCount=14, AvailableMemoryMB=91472, ConnectionCount=0 2017-02-23 21:37:17,044 WARN [main] hbase.ResourceChecker(134): Thread=762 is superior to 500 2017-02-23 21:37:17,044 WARN [main] hbase.ResourceChecker(134): OpenFileDescriptor=2167 is superior to 1024 2017-02-23 21:37:17,044 INFO [main] mapreduce.TestMultiTableInputFormat(213): Before map/reduce startup - job ScanOBBToOPP 2017-02-23 21:37:17,054 INFO [main] mapreduce.TestMultiTableInputFormat(236): scan before: {"timeRange":[0,9223372036854775807],"batch":-1,"startRow":"obb","stopRow":"opp","loadColumnFamiliesOnDemand":null,"totalColumns":1,"cacheBlocks":true,"families":{"contents":["ALL"]},"maxResultSize":-1,"maxVersions":1,"caching":-1} 2017-02-23 21:37:17,054 INFO [main] mapreduce.TestMultiTableInputFormat(236): scan before: {"timeRange":[0,9223372036854775807],"batch":-1,"startRow":"obb","stopRow":"opp","loadColumnFamiliesOnDemand":null,"totalColumns":1,"cacheBlocks":true,"families":{"contents":["ALL"]},"maxResultSize":-1,"maxVersions":1,"caching":-1} 2017-02-23 21:37:17,055 INFO [main] mapreduce.TestMultiTableInputFormat(236): scan before: {"timeRange":[0,9223372036854775807],"batch":-1,"startRow":"obb","stopRow":"opp","loadColumnFamiliesOnDemand":null,"totalColumns":1,"cacheBlocks":true,"families":{"contents":["ALL"]},"maxResultSize":-1,"maxVersions":1,"caching":-1} 2017-02-23 21:37:17,117 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.HConstants, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-common/1.1.2.2.6.0.0-SNAPSHOT/hbase-common-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:17,127 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-protocol/1.1.2.2.6.0.0-SNAPSHOT/hbase-protocol-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:17,127 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.client.Put, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-client/1.1.2.2.6.0.0-SNAPSHOT/hbase-client-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:17,128 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-hadoop-compat/1.1.2.2.6.0.0-SNAPSHOT/hbase-hadoop-compat-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:17,272 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:18,044 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:18,275 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:19,053 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:19,279 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:19,731 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 0, pendingReplications = 0. 2017-02-23 21:37:20,056 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:20,284 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:21,081 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:21,306 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:21,549 INFO [Socket Reader #1 for port 60948] ipc.Server$Connection(1573): Auth successful for appattempt_1487883618767_0003_000001 (auth:SIMPLE) 2017-02-23 21:37:21,864 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/hadoop-6889709575898132960.jar 2017-02-23 21:37:21,865 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.zookeeper.ZooKeeper, using jar /grid/0/nobody/.m2/repository/org/apache/zookeeper/zookeeper/3.4.6.2.6.0.0-SNAPSHOT/zookeeper-3.4.6.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:21,866 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class io.netty.channel.Channel, using jar /grid/0/nobody/.m2/repository/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar 2017-02-23 21:37:21,866 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class com.google.protobuf.Message, using jar /grid/0/nobody/.m2/repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2017-02-23 21:37:21,866 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class com.google.common.collect.Lists, using jar /grid/0/nobody/.m2/repository/com/google/guava/guava/12.0.1/guava-12.0.1.jar 2017-02-23 21:37:21,874 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.htrace.Trace, using jar /grid/0/nobody/.m2/repository/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar 2017-02-23 21:37:21,875 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class com.yammer.metrics.core.MetricsRegistry, using jar /grid/0/nobody/.m2/repository/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar 2017-02-23 21:37:21,875 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-common/1.1.2.2.6.0.0-SNAPSHOT/hbase-common-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:21,876 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /grid/0/nobody/.m2/repository/org/apache/hbase/hbase-common/1.1.2.2.6.0.0-SNAPSHOT/hbase-common-1.1.2.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:22,122 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:22,336 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:22,732 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 0, pendingReplications = 0. 2017-02-23 21:37:22,931 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.hbase.mapreduce.MultiTableInputFormat, using jar /grid/0/nobody/workspace/build-support/SOURCES/hbase/hbase-server/target/test-data/2b9a9b87-3f4c-430d-8fb1-df57b9b67a73/hadoop-318387561088348035.jar 2017-02-23 21:37:22,932 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.io.LongWritable, using jar /grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-common/2.7.3.2.6.0.0-SNAPSHOT/hadoop-common-2.7.3.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:22,932 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.io.Text, using jar /grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-common/2.7.3.2.6.0.0-SNAPSHOT/hadoop-common-2.7.3.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:22,933 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.7.3.2.6.0.0-SNAPSHOT/hadoop-mapreduce-client-core-2.7.3.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:22,933 DEBUG [main] mapreduce.TableMapReduceUtil(861): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /grid/0/nobody/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.7.3.2.6.0.0-SNAPSHOT/hadoop-mapreduce-client-core-2.7.3.2.6.0.0-SNAPSHOT.jar 2017-02-23 21:37:22,934 INFO [main] mapreduce.TestMultiTableInputFormat(246): Started ScanOBBToOPP 2017-02-23 21:37:23,041 WARN [main] mapreduce.JobResourceUploader(64): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2017-02-23 21:37:23,160 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:23,340 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:24,174 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:24,344 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:24,347 WARN [main] mapreduce.JobResourceUploader(171): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2017-02-23 21:37:24,375 INFO [main] zookeeper.RecoverableZooKeeper(120): Process identifier=hconnection-0x64697680 connecting to ZooKeeper ensemble=localhost:52433 2017-02-23 21:37:24,399 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(515): hconnection-0x646976800x0, quorum=localhost:52433, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2017-02-23 21:37:24,400 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(578): hconnection-0x64697680-0x15a6cc5bffa0040 connected 2017-02-23 21:37:24,401 DEBUG [main] ipc.AbstractRpcClient(105): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f2274f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2017-02-23 21:37:24,401 INFO [main] util.RegionSizeCalculator(91): Calculating region sizes for table "scantest0". 2017-02-23 21:37:24,406 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:24,406 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:37:24,735 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:24,736 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(698): Not trying to connect to xxx/xxx:36349 this server is in the failed servers list 2017-02-23 21:37:25,178 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:25,241 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:25,242 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(698): Not trying to connect to xxx/xxx:36349 this server is in the failed servers list 2017-02-23 21:37:25,349 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:25,733 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:25,733 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:26,181 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:26,249 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:26,250 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(698): Not trying to connect to xxx/xxx:36349 this server is in the failed servers list 2017-02-23 21:37:26,365 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:27,185 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:27,384 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:28,188 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:28,264 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:28,265 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:37:28,388 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:28,734 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:28,734 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:29,191 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:29,391 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:30,194 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:30,395 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:31,218 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:31,399 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:31,734 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 2 blocks at priority level 1; Total=2 Reset bookmarks? true 2017-02-23 21:37:31,735 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:32,221 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:32,269 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:32,269 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:37:32,402 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:33,224 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:33,406 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:34,227 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:34,409 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:34,735 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:34,736 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:35,229 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:35,413 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:36,233 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:36,416 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:37,236 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:37,420 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:37,736 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:37,737 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:38,239 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:38,423 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:39,242 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:39,426 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:40,245 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:40,430 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:40,737 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 2 blocks at priority level 1; Total=2 Reset bookmarks? true 2017-02-23 21:37:40,737 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:41,248 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:41,433 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:42,251 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:42,274 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:42,274 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:37:42,437 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:43,254 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:43,441 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:43,738 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:43,739 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:44,257 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:44,445 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:45,260 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:45,448 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:46,275 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:46,451 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:46,739 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:46,739 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:47,278 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:47,455 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:48,281 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:48,458 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:49,284 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:49,461 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:49,740 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 2 blocks at priority level 1; Total=2 Reset bookmarks? true 2017-02-23 21:37:49,740 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:50,287 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:50,464 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:51,290 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:51,467 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:52,293 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:52,365 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:37:52,366 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:37:52,470 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:52,740 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:52,741 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:53,296 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:53,474 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:54,300 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:54,478 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:55,304 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:55,482 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:55,741 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:37:55,742 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:56,307 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:56,485 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:57,310 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:57,489 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:58,313 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:58,492 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:58,742 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 2 blocks at priority level 1; Total=2 Reset bookmarks? true 2017-02-23 21:37:58,743 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:37:59,316 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:37:59,496 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:00,320 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:00,500 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:01,323 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:01,503 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:01,743 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:38:01,744 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:38:02,326 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:02,377 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:38:02,377 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:38:02,505 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:03,329 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:03,509 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:04,332 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:04,512 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:04,744 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:38:04,744 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:38:05,335 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:05,515 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:06,338 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:06,519 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:07,341 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:07,522 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:07,745 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 2 blocks at priority level 1; Total=2 Reset bookmarks? true 2017-02-23 21:38:07,745 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:38:08,344 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:08,526 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:09,347 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:09,529 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:10,349 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:10,532 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:10,746 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.UnderReplicatedBlocks(395): chooseUnderReplicatedBlocks selected 6 blocks at priority level 1; Total=6 Reset bookmarks? false 2017-02-23 21:38:10,746 INFO [org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$ReplicationMonitor@7d6f4421] blockmanagement.BlockManager(1594): BLOCK* neededReplications = 14, pendingReplications = 0. 2017-02-23 21:38:11,353 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:11,712 DEBUG [RS:1;xxx:40055] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:12,356 DEBUG [RS:2;xxx:60846] regionserver.HRegionServer(2324): No master found; retry 2017-02-23 21:38:12,397 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(345): Use SIMPLE authentication for service ClientService, sasl=false 2017-02-23 21:38:12,398 DEBUG [htable-pool72-t1] ipc.RpcClientImpl$Connection(710): Connecting to xxx/xxx:36349 2017-02-23 21:38:12,404 INFO [main] client.ConnectionManager$HConnectionImplementation(1687): Closing zookeeper sessionid=0x15a6cc5bffa0040 2017-02-23 21:38:12,415 DEBUG [main] ipc.RpcClientImpl(1115): Stopping rpc client