the problem is that we are having "gaps" between regions demo__users,user_8949795897,1264089193398 l2:60030 736660864 user_8949795897 user_8950697145 demo__users,user_8953502603,1263992844343 l5:60030 593335873 user_8953502603 user_8956071605 -- shell: -- get 'demo__users', "user_895158550" org.apache.hadoop.hbase.regionserver.WrongRegionException: org.apache.hadoop.hbase.regionserver.WrongRegionException: Requested row out of range for HRegion demo__users,user_8949795897,1264089193398, startKey='user_8949795897', getEndKey()='user_8950697145', row='user_895158550' at org.apache.hadoop.hbase.regionserver.HRegion.checkRow(HRegion.java:1597) at org.apache.hadoop.hbase.regionserver.HRegion.obtainRowLock(HRegion.java:1629) at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:1697) at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2362) at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1642) at sun.reflect.GeneratedMethodAccessor12.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:652) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915) -- Master logs with region that exists now, /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:45,884 INFO org.apache.hadoop.hbase.master.ServerManager: Processing MSG_REPORT_SPLIT: demo__users,user_8949795897,1263983921730: Daughters; demo__users,user_8949795897,1264089193398, demo__users,user_8950697145,1264089193398 from l2,60020,1264057760555; 1 of 3 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:45,909 INFO org.apache.hadoop.hbase.master.RegionManager: Assigning region demo__users,user_8950697145,1264089193398 to l2,60020,1264057760555 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:49,023 INFO org.apache.hadoop.hbase.master.ServerManager: Processing MSG_REPORT_PROCESS_OPEN: demo__users,user_8950697145,1264089193398 from l2,60020,1264057760555; 1 of 2 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:49,023 INFO org.apache.hadoop.hbase.master.ServerManager: Processing MSG_REPORT_OPEN: demo__users,user_8950697145,1264089193398 from l2,60020,1264057760555; 2 of 2 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:49,023 INFO org.apache.hadoop.hbase.master.RegionServerOperation: demo__users,user_8950697145,1264089193398 open on 10.72.135.6:60020 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-21:2010-01-21 07:53:49,039 INFO org.apache.hadoop.hbase.master.RegionServerOperation: Updated row demo__users,user_8950697145,1264089193398 in region .META.,,1 with startcode=1264057760555, server=10.72.135.6:60020 -- the region that begins with user_8950697145 was split on the 21st. But now, it doesn't exist anymore -- problems begin here /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:43:49,362 INFO org.apache.hadoop.hbase.master.RegionManager: Assigning region demo__users,user_8950697145,1264089193398 to l3,60020,1264661019484 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:43:52,410 INFO org.apache.hadoop.hbase.master.ServerManager: Processing MSG_REPORT_PROCESS_OPEN: demo__users,user_8950697145,1264089193398 from l3,60020,1264661019484; 6 of 19 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:43:52,413 INFO org.apache.hadoop.hbase.master.ServerManager: Processing MSG_REPORT_OPEN: demo__users,user_8950697145,1264089193398 from l3,60020,1264661019484; 16 of 19 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:46:45,007 INFO org.apache.hadoop.hbase.master.RegionServerOperation: demo__users,user_8950697145,1264089193398 open on 10.72.135.7:60020 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:46:45,010 INFO org.apache.hadoop.hbase.master.RegionServerOperation: Updated row demo__users,user_8950697145,1264089193398 in region .META.,,1 with startcode=1264661019484, server=10.72.135.7:60020 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:46:45,010 INFO org.apache.hadoop.hbase.master.RegionServerOperation: demo__users,user_8950697145,1264089193398 open on 10.72.135.7:60020 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:46:45,012 INFO org.apache.hadoop.hbase.master.RegionServerOperation: Updated row demo__users,user_8950697145,1264089193398 in region .META.,,1 with startcode=1264661019484, server=10.72.135.7:60020 /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:47:38,415 WARN org.apache.hadoop.hbase.master.HMaster: info:regioninfo is empty for row: demo__users,user_8950697145,1264089193398; has keys: info:server, info:serverstartcode /var/log/hbase/hbase-hadoop-master-l1.log.2010-01-27:2010-01-27 22:47:38,813 WARN org.apache.hadoop.hbase.master.HMaster: Removed region: demo__users,user_8950697145,1264089193398 from meta region: .META.,,1 because HRegionInfo was empty -- grepping on l2, l3, 10.72.135.6, for grep -R "user_8950697145" /var/log/hbase/ on l2 [root@l2 ~]# grep -R "user_8950697145" /var/log/hbase/hbase-hadoop-regionserver-l2.log.* /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:13,792 INFO org.apache.hadoop.hbase.regionserver.HRegion: region demo__users,user_8950697145,1264089193398/22499055 available; sequence id is 2588921043 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:13,793 INFO org.apache.hadoop.hbase.regionserver.HRegion: Closed demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:13,811 INFO org.apache.hadoop.hbase.regionserver.CompactSplitThread: region split, META updated, and report to master all successful. Old region=REGION => {NAME => 'demo__users,user_8949795897,1263983921730', STARTKEY => 'user_8949795897', ENDKEY => 'user_8952362179', ENCODED => 1858563741, OFFLINE => true, SPLIT => true, TABLE => {{NAME => 'demo__users', FAMILIES => [{NAME => 'test', COMPRESSION => 'NONE', VERSIONS => '3', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]}}, new regions: demo__users,user_8949795897,1264089193398, demo__users,user_8950697145,1264089193398. Split took 0sec /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:15,207 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: MSG_REGION_OPEN: demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:15,208 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Worker: MSG_REGION_OPEN: demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 07:53:15,285 INFO org.apache.hadoop.hbase.regionserver.HRegion: region demo__users,user_8950697145,1264089193398/22499055 available; sequence id is 2588921043 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 08:18:47,254 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on region demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-21:2010-01-21 08:18:50,593 INFO org.apache.hadoop.hbase.regionserver.HRegion: compaction completed on region demo__users,user_8950697145,1264089193398 in 3sec /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-22:2010-01-22 08:41:48,929 INFO org.apache.hadoop.hbase.regionserver.HRegion: Starting compaction on region demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-22:2010-01-22 08:41:51,942 INFO org.apache.hadoop.hbase.regionserver.HRegion: compaction completed on region demo__users,user_8950697145,1264089193398 in 3sec /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-27:2010-01-27 22:43:01,532 ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: Error closing demo__users,user_8950697145,1264089193398 there was an error closing the region pe l3 [root@l3 ~]# grep -R "user_8950697145" /var/log/hbase/hbase-hadoop-regionserver-l3.log* /var/log/hbase/hbase-hadoop-regionserver-l3.log.2010-01-27:2010-01-27 22:43:52,644 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: MSG_REGION_OPEN: demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l3.log.2010-01-27:2010-01-27 22:43:52,894 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Worker: MSG_REGION_OPEN: demo__users,user_8950697145,1264089193398 /var/log/hbase/hbase-hadoop-regionserver-l3.log.2010-01-27:2010-01-27 22:43:52,934 INFO org.apache.hadoop.hbase.regionserver.HRegion: region demo__users,user_8950697145,1264089193398/22499055 available; sequence id is 2588921043 continue on l2, /var/log/hbase/hbase-hadoop-regionserver-l2.log.2010-01-27, about 22:43 (the message about ERROR closing the region) 2010-01-27 22:43:01,532 ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: Error closing demo__users,user_8950697145,1264089193398 java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:296) at org.apache.hadoop.hdfs.DFSClient.access$900(DFSClient.java:128) at org.apache.hadoop.hdfs.DFSClient$DFSInputStream.close(DFSClient.java:1959) at java.io.FilterInputStream.close(FilterInputStream.java:155) at org.apache.hadoop.hbase.io.hfile.HFile$Reader.close(HFile.java:1056) at org.apache.hadoop.hbase.regionserver.StoreFile.close(StoreFile.java:317) at org.apache.hadoop.hbase.regionserver.Store.close(Store.java:481) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:487) at org.apache.hadoop.hbase.regionserver.HRegionServer.closeAllRegions(HRegionServer.java:1521) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:611) at java.lang.Thread.run(Thread.java:619) The hadoop file system seems to be offline The messages start here: 2010-01-27 22:43:01,341 INFO org.apache.hadoop.hbase.regionserver.LogRoller: LogRoller exiting. 2010-01-27 22:43:01,342 INFO org.apache.hadoop.hbase.regionserver.HRegionServer$MajorCompactionChecker: regionserver/10.72.135.6:60020.majorCompactionChecker exiting 2010-01-27 22:43:01,342 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: regionserver/10.72.135.6:60020.cacheFlusher exiting 2010-01-27 22:43:01,342 INFO org.apache.hadoop.hbase.regionserver.CompactSplitThread: regionserver/10.72.135.6:60020.compactor exiting 2010-01-27 22:43:01,342 ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: Error closing demo__users,user_1357901779,1263986125748 java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:296) at org.apache.hadoop.hdfs.DFSClient.access$900(DFSClient.java:128) at org.apache.hadoop.hdfs.DFSClient$DFSInputStream.close(DFSClient.java:1959) at java.io.FilterInputStream.close(FilterInputStream.java:155) at org.apache.hadoop.hbase.io.hfile.HFile$Reader.close(HFile.java:1056) at org.apache.hadoop.hbase.regionserver.StoreFile.close(StoreFile.java:317) at org.apache.hadoop.hbase.regionserver.Store.close(Store.java:481) at org.apache.hadoop.hbase.regionserver.HRegion.close(HRegion.java:487) at org.apache.hadoop.hbase.regionserver.HRegionServer.closeAllRegions(HRegionServer.java:1521) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:611) at java.lang.Thread.run(Thread.java:619) 2010-01-27 22:43:01,342 FATAL org.apache.hadoop.hbase.regionserver.HRegionServer: Shutting down HRegionServer: file system not available java.io.IOException: File system is not available at org.apache.hadoop.hbase.util.FSUtils.checkFileSystemAvailable(FSUtils.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.checkFileSystem(HRegionServer.java:848) at org.apache.hadoop.hbase.regionserver.HRegionServer.cleanup(HRegionServer.java:796) at org.apache.hadoop.hbase.regionserver.HRegionServer.closeAllRegions(HRegionServer.java:1523) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:611) at java.lang.Thread.run(Thread.java:619) Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:296) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:716) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:540) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:985) at org.apache.hadoop.hbase.util.FSUtils.checkFileSystemAvailable(FSUtils.java:115) ... 5 more 2010-01-27 22:43:01,355 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Dump of metrics: request=0.0, regions=1137, stores=1137, storefiles=1136, storefileIndexSize=251, memstoreSize=15, usedHeap=5022, maxHeap=15973, blockCacheSize=2826227152, blockCacheFree=523553840, blockCacheCount=42727, blockCacheHitRatio=1 ...........................................bunch of errors about FileSystem closes 2010-01-27 22:43:01,579 INFO org.apache.hadoop.hbase.regionserver.wal.HLog: regionserver/10.72.135.6:60020.logSyncer exiting 2010-01-27 22:43:01,586 ERROR org.apache.hadoop.hbase.regionserver.HRegionServer: Close and delete failed 2010-01-27 22:43:01,586 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: telling master that region server is shutting down at: 10.72.135.6:60020 2010-01-27 22:43:01,760 WARN org.apache.hadoop.hbase.regionserver.HRegionServer: Failed to send exiting message to master: java.io.IOException: java.io.IOException: java.util.ConcurrentModificationException at java.util.TreeMap$PrivateEntryIterator.nextEntry(TreeMap.java:1100) at java.util.TreeMap$ValueIterator.next(TreeMap.java:1145) at org.apache.hadoop.hbase.master.RegionManager.isRootServerCandidate(RegionManager.java:834) at org.apache.hadoop.hbase.master.ServerManager.removeServerInfo(ServerManager.java:641) at org.apache.hadoop.hbase.master.ServerManager.processRegionServerExit(ServerManager.java:323) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:247) at org.apache.hadoop.hbase.master.HMaster.regionServerReport(HMaster.java:729) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:648) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27) at java.lang.reflect.Constructor.newInstance(Constructor.java:513) at org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:94) at org.apache.hadoop.hbase.RemoteExceptionHandler.checkThrowable(RemoteExceptionHandler.java:48) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:634) at java.lang.Thread.run(Thread.java:619) 2010-01-27 22:43:01,761 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: stopping server at: 10.72.135.6:60020 2010-01-27 22:43:02,111 INFO org.apache.hadoop.hbase.Leases: regionserver/10.72.135.6:60020.leaseChecker closing leases 2010-01-27 22:43:02,111 INFO org.apache.hadoop.hbase.Leases: regionserver/10.72.135.6:60020.leaseChecker closed leases 2010-01-27 22:43:08,240 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: worker thread exiting 2010-01-27 22:43:08,241 INFO org.apache.zookeeper.ZooKeeper: Closing session: 0x3264f8df1780001 2010-01-27 22:43:08,241 INFO org.apache.zookeeper.ClientCnxn: Closing ClientCnxn for session: 0x3264f8df1780001 2010-01-27 22:43:08,244 INFO org.apache.zookeeper.ClientCnxn: Exception while closing send thread for session 0x3264f8df1780001 : Read error rc = -1 java.nio.DirectByteBuffer[pos=0 lim=4 cap=4] 2010-01-27 22:43:08,344 INFO org.apache.zookeeper.ClientCnxn: Disconnecting ClientCnxn for session: 0x3264f8df1780001 2010-01-27 22:43:08,344 INFO org.apache.zookeeper.ZooKeeper: Session: 0x3264f8df1780001 closed 2010-01-27 22:43:08,344 INFO org.apache.zookeeper.ClientCnxn: EventThread shut down 2010-01-27 22:43:08,344 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: regionserver/10.72.135.6:60020 exiting 2010-01-27 22:43:08,344 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Shutdown thread complete ................................................. Then it starts Wed Jan 27 22:43:15 PST 2010 Starting regionserver on l2 ulimit -n 1024 2010-01-27 22:43:16,168 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: vmInputArguments=[-Xmx16000m, -XX:+HeapDumpOnOutOfMemoryError, -XX:+UseConcMarkSweepGC, -XX:+CMSIncrementalMode, -Xdebug, -Xrunjdwp:transport=dt_socket,address=1050,server=y,suspend=n, -Dhbase.log.dir=/var/log/hbase/, -Dhbase.log.file=hbase-hadoop-regionserver-l2.log, -Dhbase.home.dir=/home/hadoop/hbase/bin/.., -Dhbase.id.str=hadoop, -Dhbase.root.logger=INFO,DRFA] 2010-01-27 22:43:16,209 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: My address is l2:60020 2010-01-27 22:43:16,290 INFO org.apache.hadoop.hbase.ipc.HBaseRpcMetrics: Initializing RPC Metrics with hostName=HRegionServer, port=60020 2010-01-27 22:43:16,386 INFO org.apache.hadoop.hbase.regionserver.MemStoreFlusher: globalMemStoreLimit=6.2g, globalMemStoreLimitLowMark=3.9g, maxHeap=15.6g 2010-01-27 22:43:16,387 INFO org.apache.hadoop.hbase.regionserver.HRegionServer: Runs every 10000000ms ................................................. then we get a bunch of errors about the file system 2010-01-27 22:44:12,302 INFO org.apache.hadoop.hdfs.DFSClient: Failed to connect to /10.72.135.6:50010, add to deadNodes and continue java.io.IOException: Got error for OP_READ_BLOCK, self=/10.72.135.6:34671, remote=/10.72.135.6:50010, for file /hbase/demo__users/1859142135/test/5562973195650977187, for block -3444385316074941249_235637 at org.apache.hadoop.hdfs.DFSClient$BlockReader.newBlockReader(DFSClient.java:1558) at org.apache.hadoop.hdfs.DFSClient$DFSInputStream.blockSeekTo(DFSClient.java:1911) at org.apache.hadoop.hdfs.DFSClient$DFSInputStream.read(DFSClient.java:2046) at java.io.DataInputStream.readFully(DataInputStream.java:178) at java.io.DataInputStream.readFully(DataInputStream.java:152) at org.apache.hadoop.hbase.io.hfile.HFile$FixedFileTrailer.deserialize(HFile.java:1338) at org.apache.hadoop.hbase.io.hfile.HFile$Reader.readTrailer(HFile.java:822) at org.apache.hadoop.hbase.io.hfile.HFile$Reader.loadFileInfo(HFile.java:767) at org.apache.hadoop.hbase.regionserver.StoreFile.open(StoreFile.java:273) at org.apache.hadoop.hbase.regionserver.StoreFile.(StoreFile.java:129) at org.apache.hadoop.hbase.regionserver.Store.loadStoreFiles(Store.java:404) at org.apache.hadoop.hbase.regionserver.Store.(Store.java:226) at org.apache.hadoop.hbase.regionserver.HRegion.instantiateHStore(HRegion.java:1506) at org.apache.hadoop.hbase.regionserver.HRegion.initialize(HRegion.java:306) at org.apache.hadoop.hbase.regionserver.HRegionServer.instantiateRegion(HRegionServer.java:1467) at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:1434) at org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1354) at java.lang.Thread.run(Thread.java:619) java.io.IOException: java.io.IOException: java.util.ConcurrentModificationException at java.util.TreeMap$NavigableSubMap$SubMapIterator.nextEntry(TreeMap.java:1572) at java.util.TreeMap$NavigableSubMap$SubMapEntryIterator.next(TreeMap.java:1620) at java.util.TreeMap$NavigableSubMap$SubMapEntryIterator.next(TreeMap.java:1614) at java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$1.next(Collections.java:1346) at java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$1.next(Collections.java:1339) at java.util.TreeMap.buildFromSorted(TreeMap.java:2372) at java.util.TreeMap.buildFromSorted(TreeMap.java:2364) at java.util.TreeMap.buildFromSorted(TreeMap.java:2322) at java.util.TreeMap.putAll(TreeMap.java:296) at org.apache.hadoop.hbase.master.RegionManager.computeNextHeaviestLoad(RegionManager.java:440) at org.apache.hadoop.hbase.master.RegionManager.assignRegionsToMultipleServers(RegionManager.java:250) at org.apache.hadoop.hbase.master.RegionManager.assignRegions(RegionManager.java:206) at org.apache.hadoop.hbase.master.ServerManager.processMsgs(ServerManager.java:462) at org.apache.hadoop.hbase.master.ServerManager.processRegionServerAllsWell(ServerManager.java:397) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:314) at org.apache.hadoop.hbase.master.HMaster.regionServerReport(HMaster.java:733) at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:652) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27) at java.lang.reflect.Constructor.newInstance(Constructor.java:513) at org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:94) at org.apache.hadoop.hbase.RemoteExceptionHandler.checkThrowable(RemoteExceptionHandler.java:48) at org.apache.hadoop.hbase.RemoteExceptionHandler.checkIOException(RemoteExceptionHandler.java:66) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:550) at java.lang.Thread.run(Thread.java:619) java.io.IOException: java.io.IOException: java.util.ConcurrentModificationException at java.util.TreeMap$PrivateEntryIterator.nextEntry(TreeMap.java:1100) at java.util.TreeMap$ValueIterator.next(TreeMap.java:1145) at org.apache.hadoop.hbase.master.RegionManager.isRootServerCandidate(RegionManager.java:834) at org.apache.hadoop.hbase.master.ServerManager.removeServerInfo(ServerManager.java:641) at org.apache.hadoop.hbase.master.ServerManager.processRegionServerExit(ServerManager.java:323) at org.apache.hadoop.hbase.master.ServerManager.regionServerReport(ServerManager.java:247) at org.apache.hadoop.hbase.master.HMaster.regionServerReport(HMaster.java:729) at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:648) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:915) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27) at java.lang.reflect.Constructor.newInstance(Constructor.java:513) at org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:94) at org.apache.hadoop.hbase.RemoteExceptionHandler.checkThrowable(RemoteExceptionHandler.java:48) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:634) at java.lang.Thread.run(Thread.java:619)