java.lang.AssertionError
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertFalse(Assert.java:68)
at org.junit.Assert.assertFalse(Assert.java:79)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:925)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
Starting DataNode 0 with dfs.data.dir: /Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data1,/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data2
Starting DataNode 1 with dfs.data.dir: /Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data3,/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data4
2013-03-20 16:03:12,647 INFO [main] hbase.ResourceChecker(157): before master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: 4 threads, 94 file descriptors 0 connections,
2013-03-20 16:03:12,865 INFO [Thread-1] hbase.HBaseTestingUtility(604): Starting up minicluster with 1 master(s) and 2 regionserver(s) and 2 datanode(s)
2013-03-20 16:03:13,647 WARN [Thread-1] impl.MetricsSystemImpl(137): Metrics system not started: Cannot locate configuration: tried hadoop-metrics2-namenode.properties, hadoop-metrics2.properties
2013-03-20 16:03:13,853 INFO [Thread-1] log.Slf4jLog(67): Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog
2013-03-20 16:03:13,921 INFO [Thread-1] log.Slf4jLog(67): jetty-6.1.26
2013-03-20 16:03:13,948 INFO [Thread-1] log.Slf4jLog(67): Extract jar:file:/Users/jzhong/.m2/repository/org/apache/hadoop/hadoop-core/1.0.4/hadoop-core-1.0.4.jar!/webapps/hdfs to /var/folders/fd/t8t76w1s4_197zykncr9b4th0000gt/T/Jetty_localhost_55431_hdfs____xx7tlo/webapp
2013-03-20 16:03:14,170 INFO [Thread-1] log.Slf4jLog(67): Started SelectChannelConnector@localhost:55431
2013-03-20 16:03:14,225 WARN [Thread-1] impl.MetricsSystemImpl(137): Metrics system not started: Cannot locate configuration: tried hadoop-metrics2-datanode.properties, hadoop-metrics2.properties
2013-03-20 16:03:14,226 WARN [Thread-1] util.MBeans(59): Hadoop:service=DataNode,name=MetricsSystem,sub=Control
javax.management.InstanceAlreadyExistsException: MXBean already registered with name Hadoop:service=NameNode,name=MetricsSystem,sub=Control
at com.sun.jmx.mbeanserver.MXBeanLookup.addReference(MXBeanLookup.java:120)
at com.sun.jmx.mbeanserver.MXBeanSupport.register(MXBeanSupport.java:143)
at com.sun.jmx.mbeanserver.MBeanSupport.preRegister2(MBeanSupport.java:183)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:941)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:917)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:312)
at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:482)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:56)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.initSystemMBean(MetricsSystemImpl.java:500)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.init(MetricsSystemImpl.java:140)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.init(DefaultMetricsSystem.java:40)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.initialize(DefaultMetricsSystem.java:50)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1520)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1496)
at org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:417)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:280)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniDFSCluster(HBaseTestingUtility.java:444)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:612)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:568)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:627)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:14,483 INFO [Thread-1] log.Slf4jLog(67): jetty-6.1.26
2013-03-20 16:03:14,494 INFO [Thread-1] log.Slf4jLog(67): Extract jar:file:/Users/jzhong/.m2/repository/org/apache/hadoop/hadoop-core/1.0.4/hadoop-core-1.0.4.jar!/webapps/datanode to /var/folders/fd/t8t76w1s4_197zykncr9b4th0000gt/T/Jetty_localhost_55435_datanode____.rcc5k3/webapp
2013-03-20 16:03:14,633 INFO [Thread-1] log.Slf4jLog(67): Started SelectChannelConnector@localhost:55435
2013-03-20 16:03:14,651 WARN [Thread-1] impl.MetricsSystemImpl(137): Metrics system not started: Cannot locate configuration: tried hadoop-metrics2-datanode.properties, hadoop-metrics2.properties
2013-03-20 16:03:14,652 WARN [Thread-1] util.MBeans(59): Hadoop:service=DataNode,name=MetricsSystem,sub=Control
javax.management.InstanceAlreadyExistsException: MXBean already registered with name Hadoop:service=NameNode,name=MetricsSystem,sub=Control
at com.sun.jmx.mbeanserver.MXBeanLookup.addReference(MXBeanLookup.java:120)
at com.sun.jmx.mbeanserver.MXBeanSupport.register(MXBeanSupport.java:143)
at com.sun.jmx.mbeanserver.MBeanSupport.preRegister2(MBeanSupport.java:183)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:941)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:917)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:312)
at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:482)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:56)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.initSystemMBean(MetricsSystemImpl.java:500)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.init(MetricsSystemImpl.java:140)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.init(DefaultMetricsSystem.java:40)
at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.initialize(DefaultMetricsSystem.java:50)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1520)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1496)
at org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:417)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:280)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniDFSCluster(HBaseTestingUtility.java:444)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:612)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:568)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:627)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:14,779 WARN [Thread-1] util.MBeans(59): Hadoop:service=DataNode,name=DataNodeInfo
javax.management.InstanceAlreadyExistsException: Hadoop:service=DataNode,name=DataNodeInfo
at com.sun.jmx.mbeanserver.Repository.addMBean(Repository.java:453)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.internal_addObject(DefaultMBeanServerInterceptor.java:1484)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerDynamicMBean(DefaultMBeanServerInterceptor.java:963)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerObject(DefaultMBeanServerInterceptor.java:917)
at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.registerMBean(DefaultMBeanServerInterceptor.java:312)
at com.sun.jmx.mbeanserver.JmxMBeanServer.registerMBean(JmxMBeanServer.java:482)
at org.apache.hadoop.metrics2.util.MBeans.register(MBeans.java:56)
at org.apache.hadoop.hdfs.server.datanode.DataNode.registerMXBean(DataNode.java:517)
at org.apache.hadoop.hdfs.server.datanode.DataNode.startDataNode(DataNode.java:393)
at org.apache.hadoop.hdfs.server.datanode.DataNode.<init>(DataNode.java:299)
at org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:1582)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1521)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:1496)
at org.apache.hadoop.hdfs.MiniDFSCluster.startDataNodes(MiniDFSCluster.java:417)
at org.apache.hadoop.hdfs.MiniDFSCluster.<init>(MiniDFSCluster.java:280)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniDFSCluster(HBaseTestingUtility.java:444)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:612)
at org.apache.hadoop.hbase.HBaseTestingUtility.startMiniCluster(HBaseTestingUtility.java:568)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:627)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:14,784 INFO [Thread-1] log.Slf4jLog(67): jetty-6.1.26
2013-03-20 16:03:14,805 INFO [Thread-1] log.Slf4jLog(67): Extract jar:file:/Users/jzhong/.m2/repository/org/apache/hadoop/hadoop-core/1.0.4/hadoop-core-1.0.4.jar!/webapps/datanode to /var/folders/fd/t8t76w1s4_197zykncr9b4th0000gt/T/Jetty_localhost_55438_datanode____t4pysg/webapp
2013-03-20 16:03:14,939 INFO [Thread-1] log.Slf4jLog(67): Started SelectChannelConnector@localhost:55438
2013-03-20 16:03:15,231 INFO [Thread-1] zookeeper.MiniZooKeeperCluster(193): Started MiniZK Cluster and connect 1 ZK server on client port: 54650
2013-03-20 16:03:15,314 DEBUG [Thread-1] util.FSUtils(398): Created version file at hdfs://localhost:55430/user/jzhong/hbase set its version at:7
2013-03-20 16:03:16,282 DEBUG [Thread-1] client.HConnectionManager(1885): Set serverside HConnection retries=100
2013-03-20 16:03:16,737 INFO [Thread-1] ipc.HBaseRpcMetrics(59): Initializing RPC Metrics with hostName=HMaster, port=55443
2013-03-20 16:03:16,957 DEBUG [Thread-1] zookeeper.ZKUtil(120): master:55443 opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:17,007 INFO [Thread-1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:21,994 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:22,018 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(350): master:55443-0x13d8a0ac7e90000 connected
2013-03-20 16:03:22,193 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: revision
2013-03-20 16:03:22,193 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: hdfsUser
2013-03-20 16:03:22,194 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: hdfsDate
2013-03-20 16:03:22,194 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: hdfsUrl
2013-03-20 16:03:22,194 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: date
2013-03-20 16:03:22,194 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: hdfsRevision
2013-03-20 16:03:22,195 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: user
2013-03-20 16:03:22,195 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: hdfsVersion
2013-03-20 16:03:22,195 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: url
2013-03-20 16:03:22,195 INFO [Thread-1] metrics.MetricsMBeanBase(109): MetricsString added: version
2013-03-20 16:03:22,196 INFO [Thread-1] metrics.MetricsMBeanBase(160): new MBeanInfo
2013-03-20 16:03:22,198 INFO [Thread-1] metrics.MetricsMBeanBase(160): new MBeanInfo
2013-03-20 16:03:22,199 INFO [Thread-1] metrics.MasterMetrics(89): Initialized
2013-03-20 16:03:22,491 DEBUG [Thread-1] client.HConnectionManager(1885): Set serverside HConnection retries=100
2013-03-20 16:03:22,509 INFO [Thread-1] ipc.HBaseRpcMetrics(59): Initializing RPC Metrics with hostName=MiniHBaseCluster$MiniHBaseClusterRegionServer, port=55445
2013-03-20 16:03:22,613 INFO [Thread-1] hfile.CacheConfig(348): Allocating LruBlockCache with maximum size 470.8m
2013-03-20 16:03:22,674 DEBUG [Thread-1] client.HConnectionManager(1885): Set serverside HConnection retries=100
2013-03-20 16:03:22,677 INFO [Thread-1] ipc.HBaseRpcMetrics(59): Initializing RPC Metrics with hostName=MiniHBaseCluster$MiniHBaseClusterRegionServer, port=55446
2013-03-20 16:03:22,685 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(425): master:55443-0x13d8a0ac7e90000 /hbase/shutdown does not exist. Watcher is set.
2013-03-20 16:03:22,687 INFO [Master:0;10.11.3.210,55443,1363820596870] master.ActiveMasterManager(143): Deleting ZNode for /hbase/backup-masters/10.11.3.210,55443,1363820596870 from backup master directory
2013-03-20 16:03:22,691 WARN [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(143): Node /hbase/backup-masters/10.11.3.210,55443,1363820596870 already deleted, and this is not a retry
2013-03-20 16:03:22,691 INFO [Master:0;10.11.3.210,55443,1363820596870] master.ActiveMasterManager(150): Master=10.11.3.210,55443,1363820596870
2013-03-20 16:03:22,735 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.SplitLogManager(175): timeout = 25000
2013-03-20 16:03:22,735 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.SplitLogManager(176): unassigned timeout = 180000
2013-03-20 16:03:22,739 INFO [Master:0;10.11.3.210,55443,1363820596870] master.SplitLogManager(833): found 0 orphan tasks and 0 rescan nodes
2013-03-20 16:03:22,769 DEBUG [Master:0;10.11.3.210,55443,1363820596870] util.FSUtils(498): Created cluster ID file at hdfs://localhost:55430/user/jzhong/hbase/hbase.id with ID: 86367a84-7905-477f-9b59-b1976e882d87
2013-03-20 16:03:22,848 INFO [Thread-1] regionserver.ShutdownHook(86): Installed shutdown hook thread: Shutdownhook:RegionServer:0;10.11.3.210,55445,1363820602585
2013-03-20 16:03:22,849 INFO [Thread-1] regionserver.ShutdownHook(86): Installed shutdown hook thread: Shutdownhook:RegionServer:1;10.11.3.210,55446,1363820602680
2013-03-20 16:03:22,869 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(120): regionserver:55445 opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:22,869 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(120): regionserver:55446 opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:22,870 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:22,870 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:22,879 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:22,881 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(350): regionserver:55445-0x13d8a0ac7e90001 connected
2013-03-20 16:03:22,883 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(423): regionserver:55445-0x13d8a0ac7e90001 Set watcher on existing znode /hbase/master
2013-03-20 16:03:22,894 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:22,895 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(425): regionserver:55445-0x13d8a0ac7e90001 /hbase/shutdown does not exist. Watcher is set.
2013-03-20 16:03:22,982 INFO [Master:0;10.11.3.210,55443,1363820596870] master.MasterFileSystem(428): BOOTSTRAP: creating ROOT and first META regions
2013-03-20 16:03:22,982 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(4167): creating HRegion -ROOT- HTD == {NAME => '-ROOT-', IS_ROOT => 'true', IS_META => 'true', FAMILIES => [{NAME => 'info', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '10', TTL => '2147483647', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '8192', ENCODE_ON_DISK => 'true', IN_MEMORY => 'false', BLOCKCACHE => 'false'}]} RootDir = hdfs://localhost:55430/user/jzhong/hbase Table name == -ROOT-
2013-03-20 16:03:22,986 DEBUG [Master:0;10.11.3.210,55443,1363820596870] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:23,423 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:23,428 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:23,468 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:23,469 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.logs/hlog.1363820603429, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:23,469 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(652): for /user/jzhong/hbase/-ROOT-/70236052/.logs/hlog.1363820603429
2013-03-20 16:03:23,470 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:23,526 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:23,559 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(477): Instantiated -ROOT-,,0.70236052
2013-03-20 16:03:23,664 INFO [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:03:23,786 INFO [StoreOpenerThread--ROOT-,,0.70236052-1] util.ChecksumType$2(77): Checksum can use java.util.zip.CRC32
2013-03-20 16:03:23,858 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(635): Onlined -ROOT-,,0.70236052; next sequenceid=1
2013-03-20 16:03:23,859 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(4167): creating HRegion .META. HTD == {NAME => '.META.', IS_META => 'true', FAMILIES => [{NAME => 'info', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '10', TTL => '2147483647', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '8192', ENCODE_ON_DISK => 'true', IN_MEMORY => 'false', BLOCKCACHE => 'false'}]} RootDir = hdfs://localhost:55430/user/jzhong/hbase Table name == .META.
2013-03-20 16:03:23,862 DEBUG [Master:0;10.11.3.210,55443,1363820596870] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/.META./1028785192/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:24,312 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:24,317 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:24,320 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:24,320 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/.META./1028785192/.logs/hlog.1363820604318, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:24,321 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(652): for /user/jzhong/hbase/.META./1028785192/.logs/hlog.1363820604318
2013-03-20 16:03:24,321 INFO [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:24,321 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:24,322 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(477): Instantiated .META.,,1.1028785192
2013-03-20 16:03:24,329 INFO [StoreOpenerThread-.META.,,1.1028785192-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:03:24,332 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(635): Onlined .META.,,1.1028785192; next sequenceid=1
2013-03-20 16:03:24,533 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(975): Closing -ROOT-,,0.70236052: disabling compactions & flushes
2013-03-20 16:03:24,533 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(997): Updates disabled for region -ROOT-,,0.70236052
2013-03-20 16:03:24,534 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1499): Started memstore flush for -ROOT-,,0.70236052, current region memstore size 392.0
2013-03-20 16:03:24,551 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1548): Finished snapshotting -ROOT-,,0.70236052, commencing wait for mvcc, flushsize=392
2013-03-20 16:03:24,551 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1558): Finished snapshotting, commencing flushing stores
2013-03-20 16:03:24,880 DEBUG [Master:0;10.11.3.210,55443,1363820596870] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/0336aa79cf35408694eeb2efd751df86 with permission=rwxrwxrwx
2013-03-20 16:03:24,897 ERROR [Master:0;10.11.3.210,55443,1363820596870] metrics.SchemaMetrics(723): Inconsistent configuration. Previous configuration for using table name in metrics: true, new configuration: false
2013-03-20 16:03:24,969 DEBUG [Master:0;10.11.3.210,55443,1363820596870] hfile.HFileWriterV2(148): Initialized with CacheConfig:enabled [cacheDataOnRead=false] [cacheDataOnWrite=false] [cacheIndexesOnWrite=false] [cacheBloomsOnWrite=false] [cacheEvictOnClose=false] [cacheCompressed=false]
2013-03-20 16:03:25,002 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.StoreFile$Writer(1047): Delete Family Bloom filter type for hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/0336aa79cf35408694eeb2efd751df86: CompoundBloomFilterWriter
2013-03-20 16:03:25,513 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.StoreFile$Writer(1267): NO General Bloom and NO DeleteFamily was added to HFile (hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/0336aa79cf35408694eeb2efd751df86)
2013-03-20 16:03:25,513 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.Store(843): Flushed , sequenceid=2, memsize=392.0, into tmp file hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/0336aa79cf35408694eeb2efd751df86
2013-03-20 16:03:25,617 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.Store(868): Renaming flushed file at hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/0336aa79cf35408694eeb2efd751df86 to hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/0336aa79cf35408694eeb2efd751df86
2013-03-20 16:03:25,626 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.Store(891): Added hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/0336aa79cf35408694eeb2efd751df86, entries=2, sequenceid=2, filesize=796.0
2013-03-20 16:03:25,629 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1632): Finished memstore flush of ~392.0/392, currentsize=0.0/0 for region -ROOT-,,0.70236052 in 1095ms, sequenceid=2, compaction requested=false
2013-03-20 16:03:25,632 INFO [StoreCloserThread--ROOT-,,0.70236052-1] regionserver.Store(712): Closed info
2013-03-20 16:03:25,632 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1045): Closed -ROOT-,,0.70236052
2013-03-20 16:03:25,632 INFO [Master:0;10.11.3.210,55443,1363820596870.logSyncer] wal.HLog$LogSyncer(1255): Master:0;10.11.3.210,55443,1363820596870.logSyncer exiting
2013-03-20 16:03:25,633 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(1017): closing hlog writer in hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.logs
2013-03-20 16:03:26,145 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(982): Moved 1 log files to /user/jzhong/hbase/-ROOT-/70236052/.oldlogs
2013-03-20 16:03:26,193 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(975): Closing .META.,,1.1028785192: disabling compactions & flushes
2013-03-20 16:03:26,193 DEBUG [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(997): Updates disabled for region .META.,,1.1028785192
2013-03-20 16:03:26,193 INFO [StoreCloserThread-.META.,,1.1028785192-1] regionserver.Store(712): Closed info
2013-03-20 16:03:26,194 INFO [Master:0;10.11.3.210,55443,1363820596870] regionserver.HRegion(1045): Closed .META.,,1.1028785192
2013-03-20 16:03:26,194 INFO [Master:0;10.11.3.210,55443,1363820596870.logSyncer] wal.HLog$LogSyncer(1255): Master:0;10.11.3.210,55443,1363820596870.logSyncer exiting
2013-03-20 16:03:26,194 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(1017): closing hlog writer in hdfs://localhost:55430/user/jzhong/hbase/.META./1028785192/.logs
2013-03-20 16:03:26,606 DEBUG [Master:0;10.11.3.210,55443,1363820596870] wal.HLog(982): Moved 1 log files to /user/jzhong/hbase/.META./1028785192/.oldlogs
2013-03-20 16:03:27,182 DEBUG [Master:0;10.11.3.210,55443,1363820596870] catalog.CatalogTracker(210): Starting catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5de867d6
2013-03-20 16:03:27,183 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(425): master:55443-0x13d8a0ac7e90000 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:03:27,184 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(425): master:55443-0x13d8a0ac7e90000 /hbase/unassigned/1028785192 does not exist. Watcher is set.
2013-03-20 16:03:27,381 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/shutdown
2013-03-20 16:03:27,381 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/shutdown
2013-03-20 16:03:27,383 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 28 byte(s) of data from znode /hbase/shutdown and set watcher; Wed Mar 20 16:03:27 PDT 2013
2013-03-20 16:03:27,384 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 28 byte(s) of data from znode /hbase/shutdown and set watcher; Wed Mar 20 16:03:27 PDT 2013
2013-03-20 16:03:27,387 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] catalog.CatalogTracker(210): Starting catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@2c0cf70f
2013-03-20 16:03:27,387 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(476): Server active/primary master; 10.11.3.210,55443,1363820596870, sessionid=0x13d8a0ac7e90000, cluster-up flag was=false
2013-03-20 16:03:27,388 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(425): regionserver:55445-0x13d8a0ac7e90001 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:03:27,389 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(425): regionserver:55445-0x13d8a0ac7e90001 /hbase/unassigned/1028785192 does not exist. Watcher is set.
2013-03-20 16:03:27,875 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:27,876 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(423): regionserver:55446 Set watcher on existing znode /hbase/master
2013-03-20 16:03:27,877 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(350): regionserver:55446-0x13d8a0ac7e90002 connected
2013-03-20 16:03:27,877 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:27,878 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(423): regionserver:55446-0x13d8a0ac7e90002 Set watcher on existing znode /hbase/shutdown
2013-03-20 16:03:27,879 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 28 byte(s) of data from znode /hbase/shutdown and set watcher; Wed Mar 20 16:03:27 PDT 2013
2013-03-20 16:03:27,880 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] catalog.CatalogTracker(210): Starting catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5b46ac5a
2013-03-20 16:03:27,881 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(425): regionserver:55446-0x13d8a0ac7e90002 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:03:27,882 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(425): regionserver:55446-0x13d8a0ac7e90002 /hbase/unassigned/1028785192 does not exist. Watcher is set.
2013-03-20 16:03:27,894 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.RecoverableZooKeeper(440): Node /hbase/online-snapshot/acquired already exists and this is not a retry
2013-03-20 16:03:27,907 INFO [Master:0;10.11.3.210,55443,1363820596870] snapshot.SnapshotManager(903): Snapshot feature is not enabled, missing log and hfile cleaners.
2013-03-20 16:03:27,929 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:27,929 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:27,986 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.MemStoreFlusher(117): globalMemStoreLimit=753.4m, globalMemStoreLimitLowMark=659.2m, maxHeap=1.8g
2013-03-20 16:03:27,986 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.MemStoreFlusher(117): globalMemStoreLimit=753.4m, globalMemStoreLimitLowMark=659.2m, maxHeap=1.8g
2013-03-20 16:03:27,988 INFO [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(440): Node /hbase/online-snapshot/acquired already exists and this is not a retry
2013-03-20 16:03:27,989 INFO [Master:0;10.11.3.210,55443,1363820596870] procedure.ZKProcedureUtil(266): Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort
2013-03-20 16:03:27,992 DEBUG [Master:0;10.11.3.210,55443,1363820596870] procedure.ZKProcedureCoordinatorRpcs(194): Starting the controller for procedure member:10.11.3.210,55443,1363820596870
2013-03-20 16:03:27,993 WARN [Master:0;10.11.3.210,55443,1363820596870] snapshot.SnapshotManager(234): Couldn't delete working snapshot directory: hdfs://localhost:55430/user/jzhong/hbase/.snapshot/.tmp
2013-03-20 16:03:28,012 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer$CompactionChecker(1284): Runs every 16mins, 40sec
2013-03-20 16:03:28,012 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer$CompactionChecker(1284): Runs every 16mins, 40sec
2013-03-20 16:03:28,074 DEBUG [Master:0;10.11.3.210,55443,1363820596870] executor.ExecutorService(192): Starting executor service name=MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870, corePoolSize=5, maxPoolSize=5
2013-03-20 16:03:28,074 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1909): Attempting connect to Master server at 10.11.3.210,55443,1363820596870
2013-03-20 16:03:28,074 DEBUG [Master:0;10.11.3.210,55443,1363820596870] executor.ExecutorService(192): Starting executor service name=MASTER_CLOSE_REGION-10.11.3.210,55443,1363820596870, corePoolSize=5, maxPoolSize=5
2013-03-20 16:03:28,074 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1909): Attempting connect to Master server at 10.11.3.210,55443,1363820596870
2013-03-20 16:03:28,075 DEBUG [Master:0;10.11.3.210,55443,1363820596870] executor.ExecutorService(192): Starting executor service name=MASTER_SERVER_OPERATIONS-10.11.3.210,55443,1363820596870, corePoolSize=3, maxPoolSize=3
2013-03-20 16:03:28,075 DEBUG [Master:0;10.11.3.210,55443,1363820596870] executor.ExecutorService(192): Starting executor service name=MASTER_META_SERVER_OPERATIONS-10.11.3.210,55443,1363820596870, corePoolSize=5, maxPoolSize=5
2013-03-20 16:03:28,075 DEBUG [Master:0;10.11.3.210,55443,1363820596870] executor.ExecutorService(192): Starting executor service name=MASTER_TABLE_OPERATIONS-10.11.3.210,55443,1363820596870, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,203 DEBUG [Master:0;10.11.3.210,55443,1363820596870] cleaner.CleanerChore(86): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2013-03-20 16:03:28,250 DEBUG [Master:0;10.11.3.210,55443,1363820596870] cleaner.CleanerChore(86): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2013-03-20 16:03:28,251 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(915): Started service threads
2013-03-20 16:03:28,251 INFO [Master:0;10.11.3.210,55443,1363820596870] master.ServerManager(629): Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2, timeout of 4500 ms, interval of 1500 ms.
2013-03-20 16:03:28,465 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1938): Connected to master at /10.11.3.210:55443
2013-03-20 16:03:28,465 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1938): Connected to master at /10.11.3.210:55443
2013-03-20 16:03:28,466 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1964): Telling master at 10.11.3.210,55443,1363820596870 that we are up with port=55446, startcode=1363820602680
2013-03-20 16:03:28,466 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1964): Telling master at 10.11.3.210,55443,1363820596870 that we are up with port=55445, startcode=1363820602585
2013-03-20 16:03:28,467 INFO [IPC Server handler 2 on 55443] master.ServerManager(274): Registering server=10.11.3.210,55446,1363820602680
2013-03-20 16:03:28,467 INFO [IPC Server handler 3 on 55443] master.ServerManager(274): Registering server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:28,473 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1036): Master passed us hostname to use. Was=10.11.3.210, Now=10.11.3.210
2013-03-20 16:03:28,473 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1036): Master passed us hostname to use. Was=10.11.3.210, Now=10.11.3.210
2013-03-20 16:03:28,473 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1043): Config from master: fs.default.name=hdfs://localhost:55430
2013-03-20 16:03:28,473 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1043): Config from master: fs.default.name=hdfs://localhost:55430
2013-03-20 16:03:28,474 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1043): Config from master: hbase.rootdir=hdfs://localhost:55430/user/jzhong/hbase
2013-03-20 16:03:28,474 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1043): Config from master: hbase.rootdir=hdfs://localhost:55430/user/jzhong/hbase
2013-03-20 16:03:28,476 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2013-03-20 16:03:28,481 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(423): master:55443-0x13d8a0ac7e90000 Set watcher on existing znode /hbase/rs/10.11.3.210,55446,1363820602680
2013-03-20 16:03:28,482 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(423): master:55443-0x13d8a0ac7e90000 Set watcher on existing znode /hbase/rs/10.11.3.210,55445,1363820602585
2013-03-20 16:03:28,482 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1345): logdir=hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680
2013-03-20 16:03:28,482 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1345): logdir=hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585
2013-03-20 16:03:28,485 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:28,485 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:28,488 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:28,488 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:28,492 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:28,492 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585/10.11.3.210%2C55445%2C1363820602585.1363820608489, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:28,494 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:28,494 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] wal.HLog(652): for /user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585/10.11.3.210%2C55445%2C1363820602585.1363820608489
2013-03-20 16:03:28,494 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680/10.11.3.210%2C55446%2C1363820602680.1363820608488, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:28,495 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:28,495 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] wal.HLog(652): for /user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680/10.11.3.210%2C55446%2C1363820602680.1363820608488
2013-03-20 16:03:28,495 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:28,505 INFO [Master:0;10.11.3.210,55443,1363820596870] master.ServerManager(646): Finished waiting for region servers count to settle; checked in 2, slept for 254 ms, expecting minimum of 1, maximum of 2, master is running.
2013-03-20 16:03:28,508 INFO [Master:0;10.11.3.210,55443,1363820596870] master.MasterFileSystem(237): Log folder hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585 belongs to an existing region server
2013-03-20 16:03:28,509 INFO [Master:0;10.11.3.210,55443,1363820596870] master.MasterFileSystem(237): Log folder hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680 belongs to an existing region server
2013-03-20 16:03:28,509 INFO [Master:0;10.11.3.210,55443,1363820596870] master.MasterFileSystem(292): No logs to split
2013-03-20 16:03:28,525 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] metrics.MetricsMBeanBase(160): new MBeanInfo
2013-03-20 16:03:28,525 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] metrics.MetricsMBeanBase(160): new MBeanInfo
2013-03-20 16:03:28,526 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] metrics.RegionServerMetrics(336): Initialized
2013-03-20 16:03:28,526 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] metrics.RegionServerMetrics(336): Initialized
2013-03-20 16:03:28,535 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_OPEN_REGION-10.11.3.210,55446,1363820602680, corePoolSize=3, maxPoolSize=3
2013-03-20 16:03:28,535 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_OPEN_REGION-10.11.3.210,55445,1363820602585, corePoolSize=3, maxPoolSize=3
2013-03-20 16:03:28,536 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_OPEN_ROOT-10.11.3.210,55446,1363820602680, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,536 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_OPEN_ROOT-10.11.3.210,55445,1363820602585, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,536 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_OPEN_META-10.11.3.210,55446,1363820602680, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,536 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_OPEN_META-10.11.3.210,55445,1363820602585, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,537 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_REGION-10.11.3.210,55446,1363820602680, corePoolSize=3, maxPoolSize=3
2013-03-20 16:03:28,537 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_REGION-10.11.3.210,55445,1363820602585, corePoolSize=3, maxPoolSize=3
2013-03-20 16:03:28,537 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,537 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_ROOT-10.11.3.210,55445,1363820602585, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,537 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_META-10.11.3.210,55446,1363820602680, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,537 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] executor.ExecutorService(192): Starting executor service name=RS_CLOSE_META-10.11.3.210,55445,1363820602585, corePoolSize=1, maxPoolSize=1
2013-03-20 16:03:28,562 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1073): Serving as 10.11.3.210,55446,1363820602680, RPC listening on 10.11.3.210/10.11.3.210:55446, sessionid=0x13d8a0ac7e90002
2013-03-20 16:03:28,563 INFO [SplitLogWorker-10.11.3.210,55446,1363820602680] regionserver.SplitLogWorker(154): SplitLogWorker 10.11.3.210,55446,1363820602680 starting
2013-03-20 16:03:28,563 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] regionserver.SplitLogWorker(154): SplitLogWorker 10.11.3.210,55445,1363820602585 starting
2013-03-20 16:03:28,563 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1073): Serving as 10.11.3.210,55445,1363820602585, RPC listening on 10.11.3.210/10.11.3.210:55445, sessionid=0x13d8a0ac7e90001
2013-03-20 16:03:28,593 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(3963): Registered RegionServer MXBean
2013-03-20 16:03:28,593 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(3963): Registered RegionServer MXBean
2013-03-20 16:03:28,593 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] procedure.ZKProcedureMemberRpcs(339): Starting procedure member '10.11.3.210,55445,1363820602585'
2013-03-20 16:03:28,593 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] procedure.ZKProcedureMemberRpcs(339): Starting procedure member '10.11.3.210,55446,1363820602680'
2013-03-20 16:03:28,594 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] procedure.ZKProcedureMemberRpcs(139): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2013-03-20 16:03:28,594 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] procedure.ZKProcedureMemberRpcs(139): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2013-03-20 16:03:28,595 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] procedure.ZKProcedureMemberRpcs(155): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2013-03-20 16:03:28,596 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] procedure.ZKProcedureMemberRpcs(155): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2013-03-20 16:03:30,529 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/unassigned/70236052 because node does not exist (not an error)
2013-03-20 16:03:30,532 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/root-region-server because node does not exist (not an error)
2013-03-20 16:03:31,541 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/root-region-server because node does not exist (not an error)
2013-03-20 16:03:31,550 INFO [Master:0;10.11.3.210,55443,1363820596870] catalog.RootLocationEditor(43): Unsetting ROOT region location in ZooKeeper
2013-03-20 16:03:31,552 WARN [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(143): Node /hbase/root-region-server already deleted, and this is not a retry
2013-03-20 16:03:31,583 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 70236052 with OFFLINE state
2013-03-20 16:03:31,593 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1677): Setting table -ROOT- to ENABLED state.
2013-03-20 16:03:31,613 WARN [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(143): Node /hbase/table92/-ROOT- already deleted, and this is not a retry
2013-03-20 16:03:31,632 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for -ROOT-,,0.70236052 so generated a random one; hri=-ROOT-,,0.70236052, src=, dest=10.11.3.210,55445,1363820602585; 2 (online=2, available=2) available servers
2013-03-20 16:03:31,632 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1698): Assigning region -ROOT-,,0.70236052 to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:31,632 INFO [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1701): Current args region:{NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,} state:-ROOT-,,0.70236052 state=OFFLINE, ts=1363820611583, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.AssignmentManager.assignRoot(AssignmentManager.java:2291)
at org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:678)
at org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:575)
at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:386)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:31,632 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.ServerManager(564): New connection to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:31,645 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:31,646 INFO [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:36,510 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3087): Regions in transition timed out: -ROOT-,,0.70236052 state=OFFLINE, ts=1363820611583, server=null
2013-03-20 16:03:36,529 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3098): Region has been OFFLINE for too long, reassigning -ROOT-,,0.70236052 to a random server
2013-03-20 16:03:36,668 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:36,669 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e90003 connected
2013-03-20 16:03:36,669 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:36,670 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90003 Set watcher on existing znode /hbase/master
2013-03-20 16:03:36,672 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:36,673 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90003 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:03:36,716 INFO [PRI IPC Server handler 1 on 55445] regionserver.HRegionServer(2863): Received request to open region: -ROOT-,,0.70236052
2013-03-20 16:03:36,717 DEBUG [PRI IPC Server handler 1 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,718 DEBUG [PRI IPC Server handler 1 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:36,720 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,720 DEBUG [PRI IPC Server handler 1 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 70236052 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,721 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,721 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=70236052/-ROOT-
2013-03-20 16:03:36,742 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 70236052 with OFFLINE state
2013-03-20 16:03:36,744 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,748 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,749 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:36,749 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:36,750 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for -ROOT-,,0.70236052 destination server is 10.11.3.210,55445,1363820602585
2013-03-20 16:03:36,750 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=70236052/-ROOT-
2013-03-20 16:03:36,750 DEBUG [pool-8-thread-1] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for -ROOT-,,0.70236052 so generated a random one; hri=-ROOT-,,0.70236052, src=, dest=10.11.3.210,55446,1363820602680; 2 (online=2, available=2) available servers
2013-03-20 16:03:36,750 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region -ROOT-,,0.70236052 to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,750 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,} state:-ROOT-,,0.70236052 state=PENDING_OPEN, ts=1363820616742, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:36,750 DEBUG [pool-8-thread-1] master.ServerManager(564): New connection to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,756 INFO [PRI IPC Server handler 1 on 55446] regionserver.HRegionServer(2863): Received request to open region: -ROOT-,,0.70236052
2013-03-20 16:03:36,756 DEBUG [PRI IPC Server handler 1 on 55446] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 70236052/-ROOT- from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,758 DEBUG [PRI IPC Server handler 1 on 55446] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:36,758 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}
2013-03-20 16:03:36,759 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:36,759 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,759 DEBUG [PRI IPC Server handler 1 on 55446] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 70236052 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,762 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(4239): Opening region: {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}
2013-03-20 16:03:36,762 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,762 INFO [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:36,762 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=70236052/-ROOT-
2013-03-20 16:03:36,779 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(477): Instantiated -ROOT-,,0.70236052
2013-03-20 16:03:36,779 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated -ROOT-,,0.70236052
2013-03-20 16:03:36,810 INFO [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:03:36,848 DEBUG [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(448): loaded hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/0336aa79cf35408694eeb2efd751df86, isReference=false, isBulkLoadResult=false, seqid=2, majorCompaction=false
2013-03-20 16:03:36,850 INFO [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(635): Onlined -ROOT-,,0.70236052; next sequenceid=3
2013-03-20 16:03:36,851 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,852 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,876 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,877 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,878 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,878 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=70236052/-ROOT-
2013-03-20 16:03:36,914 INFO [PostOpenDeployTasks:70236052] regionserver.HRegionServer(1729): Post open deploy tasks for region=-ROOT-,,0.70236052, daughter=false
2013-03-20 16:03:36,914 INFO [PostOpenDeployTasks:70236052] catalog.RootLocationEditor(62): Setting ROOT region location in ZooKeeper as 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,916 INFO [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:03:36,917 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90003 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:03:36,917 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:03:36,917 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:03:36,918 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:03:36,923 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,925 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,926 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,927 INFO [PostOpenDeployTasks:70236052] regionserver.HRegionServer(1754): Done with post open deploy task for region=-ROOT-,,0.70236052, daughter=false
2013-03-20 16:03:36,927 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,928 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:36,931 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:36,933 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,933 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:36,933 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}, server: 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,934 DEBUG [RS_OPEN_ROOT-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(135): Opened -ROOT-,,0.70236052 on server:10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,936 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:36,936 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55446,1363820602680, region=70236052/-ROOT-
2013-03-20 16:03:36,937 DEBUG [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(448): loaded hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/0336aa79cf35408694eeb2efd751df86, isReference=false, isBulkLoadResult=false, seqid=2, majorCompaction=false
2013-03-20 16:03:36,940 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined -ROOT-,,0.70236052; next sequenceid=3
2013-03-20 16:03:36,940 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:36,941 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:36,941 WARN [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(777): regionserver:55445-0x13d8a0ac7e90001 Attempt to transition the unassigned node for 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 5 not the expected version 1
2013-03-20 16:03:36,941 WARN [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(385): Failed refreshing OPENING; region=70236052, context=post_region_open
2013-03-20 16:03:36,942 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(975): Closing -ROOT-,,0.70236052: disabling compactions & flushes
2013-03-20 16:03:36,942 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(997): Updates disabled for region -ROOT-,,0.70236052
2013-03-20 16:03:36,943 INFO [StoreCloserThread--ROOT-,,0.70236052-1] regionserver.Store(712): Closed info
2013-03-20 16:03:36,943 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1045): Closed -ROOT-,,0.70236052
2013-03-20 16:03:36,943 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(301): Opening of region {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,} failed, marking as FAILED_OPEN in ZK
2013-03-20 16:03:36,943 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_FAILED_OPEN
2013-03-20 16:03:36,945 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:36,945 WARN [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(799): regionserver:55445-0x13d8a0ac7e90001 Attempt to transition the unassigned node for 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state RS_ZK_REGION_OPENED set by the server 10.11.3.210,55445,1363820602585
2013-03-20 16:03:36,945 WARN [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(308): Unable to mark region {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,} as FAILED_OPEN. It's likely that the master already timed out this open attempt, and thus another RS already has the region.
2013-03-20 16:03:36,964 INFO [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] handler.OpenedRegionHandler(145): Handling OPENED event for -ROOT-,,0.70236052 from 10.11.3.210,55446,1363820602680; deleting unassigned node
2013-03-20 16:03:36,965 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 70236052 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:36,966 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:36,968 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:03:36,968 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region -ROOT-,,0.70236052 has been deleted.
2013-03-20 16:03:36,968 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 70236052 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:36,968 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region -ROOT-,,0.70236052 that was online on 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,969 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,969 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(692): -ROOT- assigned=1, rit=false, location=10.11.3.210,55446,1363820602680
2013-03-20 16:03:36,970 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/unassigned/1028785192 because node does not exist (not an error)
2013-03-20 16:03:37,106 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,107 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,132 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,133 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,601 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,601 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,604 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,604 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,659 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,660 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,664 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,665 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,720 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,721 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,723 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,723 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,779 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,779 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,782 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,782 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,837 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,837 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,839 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,840 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,894 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,894 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,896 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,896 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,951 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,951 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,953 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:37,953 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,008 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,008 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,010 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,010 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,013 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,014 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,015 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,016 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,017 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 1028785192 with OFFLINE state
2013-03-20 16:03:38,019 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,019 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,019 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,020 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:38,021 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:38,021 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:38,021 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=1028785192/.META.
2013-03-20 16:03:38,022 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1677): Setting table .META. to ENABLED state.
2013-03-20 16:03:38,023 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:38,026 WARN [Master:0;10.11.3.210,55443,1363820596870] zookeeper.RecoverableZooKeeper(143): Node /hbase/table92/.META. already deleted, and this is not a retry
2013-03-20 16:03:38,027 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for .META.,,1.1028785192 so generated a random one; hri=.META.,,1.1028785192, src=, dest=10.11.3.210,55445,1363820602585; 2 (online=2, available=2) available servers
2013-03-20 16:03:38,028 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1698): Assigning region .META.,,1.1028785192 to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,028 INFO [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(1701): Current args region:{NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} state:.META.,,1.1028785192 state=OFFLINE, ts=1363820618017, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.AssignmentManager.assignMeta(AssignmentManager.java:2304)
at org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:707)
at org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:575)
at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:386)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:38,028 INFO [PRI IPC Server handler 2 on 55445] regionserver.HRegionServer(2863): Received request to open region: .META.,,1.1028785192
2013-03-20 16:03:38,028 DEBUG [PRI IPC Server handler 2 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 1028785192/.META. from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:38,030 DEBUG [PRI IPC Server handler 2 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192; data=region=.META.,,1, origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:38,031 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,031 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,031 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,032 DEBUG [PRI IPC Server handler 2 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 1028785192 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:38,032 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,033 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,033 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=1028785192/.META.
2013-03-20 16:03:38,035 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:38,035 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,035 INFO [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:38,036 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated .META.,,1.1028785192
2013-03-20 16:03:38,036 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,037 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,037 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,039 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,039 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,040 INFO [StoreOpenerThread-.META.,,1.1028785192-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:03:38,043 INFO [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined .META.,,1.1028785192; next sequenceid=1
2013-03-20 16:03:38,044 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:38,045 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192; data=region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,046 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,046 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,046 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,047 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:38,047 INFO [PostOpenDeployTasks:1028785192] regionserver.HRegionServer(1729): Post open deploy tasks for region=.META.,,1.1028785192, daughter=false
2013-03-20 16:03:38,048 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,048 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,048 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,048 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=1028785192/.META.
2013-03-20 16:03:38,049 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,067 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:38,068 INFO [PostOpenDeployTasks:1028785192] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:38,070 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:38,071 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(1597): hconnection Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:38,071 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e90004 connected
2013-03-20 16:03:38,072 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90004 Set watcher on existing znode /hbase/master
2013-03-20 16:03:38,073 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:38,074 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90004 Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:03:38,075 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,077 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,078 DEBUG [PostOpenDeployTasks:1028785192] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@46f5331a; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,080 DEBUG [PostOpenDeployTasks:1028785192] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,080 DEBUG [PostOpenDeployTasks:1028785192] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@46f5331a; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,094 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,094 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,096 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,097 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,151 INFO [PostOpenDeployTasks:1028785192] catalog.MetaEditor(297): Updated row .META.,,1.1028785192 with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,153 INFO [PostOpenDeployTasks:1028785192] regionserver.HRegionServer(1754): Done with post open deploy task for region=.META.,,1.1028785192, daughter=false
2013-03-20 16:03:38,153 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,153 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 1028785192/.META. from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:38,153 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,155 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192; data=region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:38,156 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,156 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,156 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,157 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,157 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,157 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 1028785192 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:38,157 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,157 DEBUG [RS_OPEN_META-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(135): Opened .META.,,1.1028785192 on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,158 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:38,158 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:38,159 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=1028785192/.META.
2013-03-20 16:03:38,159 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:38,159 INFO [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] handler.OpenedRegionHandler(145): Handling OPENED event for .META.,,1.1028785192 from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:38,160 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 1028785192 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:38,160 DEBUG [Master:0;10.11.3.210,55443,1363820596870] catalog.CatalogTracker(483): Set new cached META location: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,160 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192 and set watcher; region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:38,161 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 56 byte(s) of data from znode /hbase/unassigned/1028785192; data=region=.META.,,1, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:38,162 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,163 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,163 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region .META.,,1.1028785192 has been deleted.
2013-03-20 16:03:38,163 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/1028785192
2013-03-20 16:03:38,163 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(719): .META. assigned=2, rit=false, location=10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,163 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] catalog.CatalogTracker(471): Current cached META location, null, is not valid, resetting
2013-03-20 16:03:38,163 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 1028785192 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:38,163 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] catalog.CatalogTracker(471): Current cached META location, null, is not valid, resetting
2013-03-20 16:03:38,163 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region .META.,,1.1028785192 that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:38,164 DEBUG [Thread-1-EventThread] catalog.CatalogTracker(471): Current cached META location, 10.11.3.210,55445,1363820602585, is not valid, resetting
2013-03-20 16:03:38,184 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,184 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,207 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(90): Creating scanner over -ROOT- starting at key ''
2013-03-20 16:03:38,207 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:38,221 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,221 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,259 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(185): Finished with scanning at {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}
2013-03-20 16:03:38,260 INFO [Master:0;10.11.3.210,55443,1363820596870] catalog.MetaMigrationRemovingHTD(224): Meta version=0; migrated=true
2013-03-20 16:03:38,260 INFO [Master:0;10.11.3.210,55443,1363820596870] catalog.MetaMigrationRemovingHTD(245): ROOT/Meta already up-to date with new HRI.
2013-03-20 16:03:38,302 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90003 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,302 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@11da5362; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:38,349 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:03:38,350 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:03:38,350 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:38,354 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:38,355 INFO [Master:0;10.11.3.210,55443,1363820596870] master.AssignmentManager(470): Clean cluster startup. Assigning userregions
2013-03-20 16:03:38,355 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKAssign(530): master:55443-0x13d8a0ac7e90000 Deleting any existing unassigned nodes
2013-03-20 16:03:38,359 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:03:38,359 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:38,363 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:38,371 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:03:38,371 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:38,373 DEBUG [Master:0;10.11.3.210,55443,1363820596870] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:38,400 DEBUG [10.11.3.210,55443,1363820596870-CatalogJanitor] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:03:38,400 DEBUG [10.11.3.210,55443,1363820596870-CatalogJanitor] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:38,402 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(1955): Registered HMaster MXBean
2013-03-20 16:03:38,402 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(607): Master has completed initialization
2013-03-20 16:03:38,403 DEBUG [10.11.3.210,55443,1363820596870-CatalogJanitor] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:38,403 DEBUG [10.11.3.210,55443,1363820596870-CatalogJanitor] master.CatalogJanitor(156): Scanned 0 catalog row(s) and gc'd 0 unreferenced parent region(s)
2013-03-20 16:03:38,480 DEBUG [Thread-1] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:38,481 INFO [Thread-1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:43,485 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:43,487 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:43,487 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e90005 connected
2013-03-20 16:03:43,488 DEBUG [Thread-1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90005 Set watcher on existing znode /hbase/master
2013-03-20 16:03:43,489 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90005 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:43,494 DEBUG [Thread-1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90005 Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:03:43,495 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90005 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:43,498 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90005 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:43,498 DEBUG [Thread-1] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@49e61582; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:43,505 DEBUG [Thread-1] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:03:43,506 DEBUG [Thread-1] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:03:43,506 DEBUG [Thread-1] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:03:43,514 DEBUG [Thread-1] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:03:43,615 INFO [Thread-1] client.HConnectionManager$HConnectionImplementation(1785): Closed zookeeper sessionid=0x13d8a0ac7e90005
2013-03-20 16:03:43,655 DEBUG [Thread-1] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:43,656 INFO [Thread-1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:43,659 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:43,660 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e90006 connected
2013-03-20 16:03:43,660 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:43,662 DEBUG [Thread-1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90006 Set watcher on existing znode /hbase/master
2013-03-20 16:03:43,663 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:43,664 DEBUG [Thread-1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90006 Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:03:43,665 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:43,677 INFO [Thread-1] hbase.HBaseTestingUtility(661): Minicluster is up
2013-03-20 16:03:43,677 INFO [Thread-1] master.TestMasterFailover(951):
Cluster started
2013-03-20 16:03:43,689 DEBUG [Thread-1] zookeeper.ZKUtil(120): unittest opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:43,690 INFO [Thread-1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:43,703 INFO [Thread-1] master.HMaster(1138): BalanceSwitch=false
2013-03-20 16:03:43,704 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:43,705 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(350): unittest-0x13d8a0ac7e90007 connected
2013-03-20 16:03:43,771 INFO [Thread-1] regionserver.HRegion(4167): creating HRegion enabledTable HTD == {NAME => 'enabledTable', FAMILIES => [{NAME => 'family', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '3', TTL => '2147483647', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', ENCODE_ON_DISK => 'true', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]} RootDir = hdfs://localhost:55430/user/jzhong/hbase Table name == enabledTable
2013-03-20 16:03:43,775 DEBUG [Thread-1] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/18498a82920209c1a01bf0fa403b2c0c/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:44,191 INFO [Thread-1] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:44,196 INFO [Thread-1] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:44,198 DEBUG [Thread-1] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:44,198 DEBUG [Thread-1] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/18498a82920209c1a01bf0fa403b2c0c/.logs/hlog.1363820624196, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:44,198 INFO [Thread-1] wal.HLog(652): for /user/jzhong/hbase/enabledTable/18498a82920209c1a01bf0fa403b2c0c/.logs/hlog.1363820624196
2013-03-20 16:03:44,198 INFO [Thread-1] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:44,199 INFO [Thread-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:44,199 DEBUG [Thread-1] regionserver.HRegion(477): Instantiated enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.
2013-03-20 16:03:44,205 INFO [StoreOpenerThread-enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:44,208 INFO [Thread-1] regionserver.HRegion(635): Onlined enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.; next sequenceid=1
2013-03-20 16:03:44,208 DEBUG [Thread-1] regionserver.HRegion(975): Closing enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.: disabling compactions & flushes
2013-03-20 16:03:44,208 DEBUG [Thread-1] regionserver.HRegion(997): Updates disabled for region enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.
2013-03-20 16:03:44,209 INFO [StoreCloserThread-enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:44,209 INFO [Thread-1] regionserver.HRegion(1045): Closed enabledTable,,1363820623745.18498a82920209c1a01bf0fa403b2c0c.
2013-03-20 16:03:44,209 INFO [Thread-1.logSyncer] wal.HLog$LogSyncer(1255): Thread-1.logSyncer exiting
2013-03-20 16:03:44,209 DEBUG [Thread-1] wal.HLog(1017): closing hlog writer in hdfs://localhost:55430/user/jzhong/hbase/enabledTable/18498a82920209c1a01bf0fa403b2c0c/.logs
2013-03-20 16:03:44,665 DEBUG [Thread-1] wal.HLog(982): Moved 1 log files to /user/jzhong/hbase/enabledTable/18498a82920209c1a01bf0fa403b2c0c/.oldlogs
2013-03-20 16:03:44,669 DEBUG [Thread-1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:44,669 DEBUG [Thread-1] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@76612ef6; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:44,674 DEBUG [Thread-1] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:03:44,686 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,,1363820624674.456db60834cd049ce89711613277327e.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => 456db60834cd049ce89711613277327e,}
2013-03-20 16:03:44,690 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 22698c7f211462a832f422e476392633,}
2013-03-20 16:03:44,692 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}
2013-03-20 16:03:44,694 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}
2013-03-20 16:03:44,695 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 595e6cadb10b50694b52a5e6725b4de6,}
2013-03-20 16:03:44,697 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,eee,1363820624696.2a4e0f9d90178bbfecf09633a6933dfa.', STARTKEY => 'eee', ENDKEY => 'fff', ENCODED => 2a4e0f9d90178bbfecf09633a6933dfa,}
2013-03-20 16:03:44,700 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,fff,1363820624698.2ddca7a4038739a14492abc07a636d93.', STARTKEY => 'fff', ENDKEY => 'ggg', ENCODED => 2ddca7a4038739a14492abc07a636d93,}
2013-03-20 16:03:44,702 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,ggg,1363820624700.e90cd915ccae23af895b832a4dbc17d7.', STARTKEY => 'ggg', ENDKEY => 'hhh', ENCODED => e90cd915ccae23af895b832a4dbc17d7,}
2013-03-20 16:03:44,704 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,hhh,1363820624702.3383249d8093141c26266486bef59f3c.', STARTKEY => 'hhh', ENDKEY => 'iii', ENCODED => 3383249d8093141c26266486bef59f3c,}
2013-03-20 16:03:44,706 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,iii,1363820624704.d2428d5f096ddcafeb05949a84963f6d.', STARTKEY => 'iii', ENDKEY => 'jjj', ENCODED => d2428d5f096ddcafeb05949a84963f6d,}
2013-03-20 16:03:44,707 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'enabledTable,jjj,1363820624706.828029ef2e0b58f76fbea0de565e9274.', STARTKEY => 'jjj', ENDKEY => '', ENCODED => 828029ef2e0b58f76fbea0de565e9274,}
2013-03-20 16:03:44,721 INFO [Thread-1] regionserver.HRegion(4167): creating HRegion disabledTable HTD == {NAME => 'disabledTable', FAMILIES => [{NAME => 'family', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', COMPRESSION => 'NONE', VERSIONS => '3', TTL => '2147483647', MIN_VERSIONS => '0', KEEP_DELETED_CELLS => 'false', BLOCKSIZE => '65536', ENCODE_ON_DISK => 'true', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]} RootDir = hdfs://localhost:55430/user/jzhong/hbase Table name == disabledTable
2013-03-20 16:03:44,724 DEBUG [Thread-1] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/4099eeab5e33d9d80750c2e6b213f340/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,137 INFO [Thread-1] wal.HLog(463): FileSystem doesn't support getDefaultBlockSize
2013-03-20 16:03:45,169 INFO [Thread-1] wal.HLog(427): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms
2013-03-20 16:03:45,183 DEBUG [Thread-1] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:03:45,183 DEBUG [Thread-1] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/4099eeab5e33d9d80750c2e6b213f340/.logs/hlog.1363820625170, syncFs=true, hflush=false, compression=false
2013-03-20 16:03:45,183 INFO [Thread-1] wal.HLog(652): for /user/jzhong/hbase/disabledTable/4099eeab5e33d9d80750c2e6b213f340/.logs/hlog.1363820625170
2013-03-20 16:03:45,184 INFO [Thread-1] wal.HLog(506): Using getNumCurrentReplicas--HDFS-826
2013-03-20 16:03:45,184 INFO [Thread-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,184 DEBUG [Thread-1] regionserver.HRegion(477): Instantiated disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.
2013-03-20 16:03:45,191 INFO [StoreOpenerThread-disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,195 INFO [Thread-1] regionserver.HRegion(635): Onlined disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.; next sequenceid=1
2013-03-20 16:03:45,196 DEBUG [Thread-1] regionserver.HRegion(975): Closing disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.: disabling compactions & flushes
2013-03-20 16:03:45,196 DEBUG [Thread-1] regionserver.HRegion(997): Updates disabled for region disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.
2013-03-20 16:03:45,196 INFO [StoreCloserThread-disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:45,197 INFO [Thread-1] regionserver.HRegion(1045): Closed disabledTable,,1363820624721.4099eeab5e33d9d80750c2e6b213f340.
2013-03-20 16:03:45,197 INFO [Thread-1.logSyncer] wal.HLog$LogSyncer(1255): Thread-1.logSyncer exiting
2013-03-20 16:03:45,197 DEBUG [Thread-1] wal.HLog(1017): closing hlog writer in hdfs://localhost:55430/user/jzhong/hbase/disabledTable/4099eeab5e33d9d80750c2e6b213f340/.logs
2013-03-20 16:03:45,611 DEBUG [Thread-1] wal.HLog(982): Moved 1 log files to /user/jzhong/hbase/disabledTable/4099eeab5e33d9d80750c2e6b213f340/.oldlogs
2013-03-20 16:03:45,616 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => f62bc1775c460d6926da67fdbe4875c2,}
2013-03-20 16:03:45,619 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 4e209070c1b4810e530b1e749a5db6a1,}
2013-03-20 16:03:45,621 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 2900df9d44e0e7dff2a11e46ea809cea,}
2013-03-20 16:03:45,625 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,}
2013-03-20 16:03:45,627 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 1d9bcd12253b489d89684f8633af577a,}
2013-03-20 16:03:45,629 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,eee,1363820625628.b2b4d44172d9b4b915784938ffa5b554.', STARTKEY => 'eee', ENDKEY => 'fff', ENCODED => b2b4d44172d9b4b915784938ffa5b554,}
2013-03-20 16:03:45,632 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,fff,1363820625630.a823f0c9e9f93d1c1869aa0694d909d5.', STARTKEY => 'fff', ENDKEY => 'ggg', ENCODED => a823f0c9e9f93d1c1869aa0694d909d5,}
2013-03-20 16:03:45,634 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,ggg,1363820625632.7be9f84106d3cb4303831b1aed868843.', STARTKEY => 'ggg', ENDKEY => 'hhh', ENCODED => 7be9f84106d3cb4303831b1aed868843,}
2013-03-20 16:03:45,637 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,hhh,1363820625634.1e926241c785805f2e85cd7fe855c59f.', STARTKEY => 'hhh', ENDKEY => 'iii', ENCODED => 1e926241c785805f2e85cd7fe855c59f,}
2013-03-20 16:03:45,639 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,iii,1363820625637.0446dbe5de2b1fc0f6dc5fe104d3c9bb.', STARTKEY => 'iii', ENDKEY => 'jjj', ENCODED => 0446dbe5de2b1fc0f6dc5fe104d3c9bb,}
2013-03-20 16:03:45,641 INFO [Thread-1] hbase.HBaseTestingUtility(1305): createMultiRegionsInMeta: inserted {NAME => 'disabledTable,jjj,1363820625639.1e3107abecaed7fa08e7f8b23772f051.', STARTKEY => 'jjj', ENDKEY => '', ENCODED => 1e3107abecaed7fa08e7f8b23772f051,}
2013-03-20 16:03:45,641 INFO [Thread-1] master.TestMasterFailover(951):
Regions in META have been created
2013-03-20 16:03:45,642 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 456db60834cd049ce89711613277327e with OFFLINE state
2013-03-20 16:03:45,645 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,647 DEBUG [Thread-1] master.AssignmentManager(1677): Setting table enabledTable to ENABLED state.
2013-03-20 16:03:45,652 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,652 DEBUG [Finalizer] client.HConnectionManager$HConnectionImplementation(1822): The connection to null was closed by the finalize method.
2013-03-20 16:03:45,699 WARN [Thread-1] zookeeper.RecoverableZooKeeper(143): Node /hbase/table92/enabledTable already deleted, and this is not a retry
2013-03-20 16:03:45,752 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for enabledTable,,1363820624674.456db60834cd049ce89711613277327e. destination server is 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,752 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region enabledTable,,1363820624674.456db60834cd049ce89711613277327e.; plan=hri=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., src=, dest=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,752 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region enabledTable,,1363820624674.456db60834cd049ce89711613277327e. to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,753 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,,1363820624674.456db60834cd049ce89711613277327e.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => 456db60834cd049ce89711613277327e,} state:enabledTable,,1363820624674.456db60834cd049ce89711613277327e. state=OFFLINE, ts=1363820625642, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:723)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,754 INFO [PRI IPC Server handler 3 on 55445] regionserver.HRegionServer(2863): Received request to open region: enabledTable,,1363820624674.456db60834cd049ce89711613277327e.
2013-03-20 16:03:45,777 DEBUG [PRI IPC Server handler 3 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 456db60834cd049ce89711613277327e from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,779 DEBUG [PRI IPC Server handler 3 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e; data=region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,780 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/456db60834cd049ce89711613277327e
2013-03-20 16:03:45,781 DEBUG [PRI IPC Server handler 3 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 456db60834cd049ce89711613277327e from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,784 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,,1363820624674.456db60834cd049ce89711613277327e.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => 456db60834cd049ce89711613277327e,}
2013-03-20 16:03:45,784 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 22698c7f211462a832f422e476392633 with OFFLINE state
2013-03-20 16:03:45,785 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,785 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,785 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=456db60834cd049ce89711613277327e
2013-03-20 16:03:45,785 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated enabledTable,,1363820624674.456db60834cd049ce89711613277327e.
2013-03-20 16:03:45,787 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/456db60834cd049ce89711613277327e/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,787 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,789 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. destination server is 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,789 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.; plan=hri=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., src=, dest=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,789 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,789 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 22698c7f211462a832f422e476392633,} state:enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. state=OFFLINE, ts=1363820625784, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:723)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,790 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,790 INFO [PRI IPC Server handler 4 on 55445] regionserver.HRegionServer(2863): Received request to open region: enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.
2013-03-20 16:03:45,791 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,792 DEBUG [PRI IPC Server handler 4 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 22698c7f211462a832f422e476392633 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,793 DEBUG [PRI IPC Server handler 4 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633; data=region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,797 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/22698c7f211462a832f422e476392633
2013-03-20 16:03:45,797 DEBUG [PRI IPC Server handler 4 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 22698c7f211462a832f422e476392633 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,799 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 22698c7f211462a832f422e476392633,}
2013-03-20 16:03:45,799 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for f62bc1775c460d6926da67fdbe4875c2 with OFFLINE state
2013-03-20 16:03:45,800 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,800 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,800 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=22698c7f211462a832f422e476392633
2013-03-20 16:03:45,800 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(477): Instantiated enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.
2013-03-20 16:03:45,802 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/22698c7f211462a832f422e476392633/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,802 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,804 DEBUG [Thread-1] master.AssignmentManager(1677): Setting table disabledTable to ENABLED state.
2013-03-20 16:03:45,805 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,806 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,806 INFO [StoreOpenerThread-enabledTable,,1363820624674.456db60834cd049ce89711613277327e.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,807 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,810 WARN [Thread-1] zookeeper.RecoverableZooKeeper(143): Node /hbase/table92/disabledTable already deleted, and this is not a retry
2013-03-20 16:03:45,812 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined enabledTable,,1363820624674.456db60834cd049ce89711613277327e.; next sequenceid=1
2013-03-20 16:03:45,812 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 456db60834cd049ce89711613277327e from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,813 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. destination server is 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,813 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.; plan=hri=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., src=, dest=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,813 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,813 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => f62bc1775c460d6926da67fdbe4875c2,} state:disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. state=OFFLINE, ts=1363820625799, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
2013-03-20 16:03:45,814 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e; data=region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:728)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,815 INFO [PRI IPC Server handler 5 on 55445] regionserver.HRegionServer(2863): Received request to open region: disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:03:45,815 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/456db60834cd049ce89711613277327e
2013-03-20 16:03:45,815 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 456db60834cd049ce89711613277327e from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,816 INFO [PostOpenDeployTasks:456db60834cd049ce89711613277327e] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., daughter=false
2013-03-20 16:03:45,816 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,817 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=456db60834cd049ce89711613277327e
2013-03-20 16:03:45,820 DEBUG [PostOpenDeployTasks:456db60834cd049ce89711613277327e] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,820 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node f62bc1775c460d6926da67fdbe4875c2 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,820 DEBUG [PostOpenDeployTasks:456db60834cd049ce89711613277327e] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@46f5331a; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,822 INFO [StoreOpenerThread-enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,823 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,824 DEBUG [PostOpenDeployTasks:456db60834cd049ce89711613277327e] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:03:45,824 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,825 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node f62bc1775c460d6926da67fdbe4875c2 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,829 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(4239): Opening region: {NAME => 'disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => f62bc1775c460d6926da67fdbe4875c2,}
2013-03-20 16:03:45,829 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 4e209070c1b4810e530b1e749a5db6a1 with OFFLINE state
2013-03-20 16:03:45,829 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,830 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(635): Onlined enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.; next sequenceid=1
2013-03-20 16:03:45,830 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,830 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 22698c7f211462a832f422e476392633 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,830 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,830 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(477): Instantiated disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:03:45,832 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633; data=region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,832 INFO [PostOpenDeployTasks:456db60834cd049ce89711613277327e] catalog.MetaEditor(297): Updated row enabledTable,,1363820624674.456db60834cd049ce89711613277327e. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,832 INFO [PostOpenDeployTasks:456db60834cd049ce89711613277327e] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., daughter=false
2013-03-20 16:03:45,832 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,833 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 456db60834cd049ce89711613277327e from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,832 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/f62bc1775c460d6926da67fdbe4875c2/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,834 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 22698c7f211462a832f422e476392633 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,835 INFO [PostOpenDeployTasks:22698c7f211462a832f422e476392633] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., daughter=false
2013-03-20 16:03:45,837 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e; data=region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,838 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. destination server is 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,838 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.; plan=hri=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., src=, dest=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,838 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,838 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,838 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 4e209070c1b4810e530b1e749a5db6a1,} state:disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. state=OFFLINE, ts=1363820625829, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:728)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,839 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 456db60834cd049ce89711613277327e from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,839 INFO [PRI IPC Server handler 9 on 55445] regionserver.HRegionServer(2863): Received request to open region: disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:03:45,839 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,,1363820624674.456db60834cd049ce89711613277327e.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => 456db60834cd049ce89711613277327e,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,840 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,839 INFO [PostOpenDeployTasks:22698c7f211462a832f422e476392633] catalog.MetaEditor(297): Updated row enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,840 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(135): Opened enabledTable,,1363820624674.456db60834cd049ce89711613277327e. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,840 INFO [PostOpenDeployTasks:22698c7f211462a832f422e476392633] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., daughter=false
2013-03-20 16:03:45,841 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 22698c7f211462a832f422e476392633 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,841 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e209070c1b4810e530b1e749a5db6a1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,841 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,842 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633; data=region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,845 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,846 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/22698c7f211462a832f422e476392633
2013-03-20 16:03:45,846 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,847 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,847 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=22698c7f211462a832f422e476392633
2013-03-20 16:03:45,849 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/456db60834cd049ce89711613277327e
2013-03-20 16:03:45,849 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 22698c7f211462a832f422e476392633 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,849 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 22698c7f211462a832f422e476392633,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,849 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(135): Opened enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,850 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e209070c1b4810e530b1e749a5db6a1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,850 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => 'disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 4e209070c1b4810e530b1e749a5db6a1,}
2013-03-20 16:03:45,851 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e and set watcher; region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,851 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,851 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 51bb507664c357c5303e3d230413e1c1 with OFFLINE state
2013-03-20 16:03:45,851 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=456db60834cd049ce89711613277327e
2013-03-20 16:03:45,851 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:03:45,851 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/22698c7f211462a832f422e476392633
2013-03-20 16:03:45,855 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,,1363820624674.456db60834cd049ce89711613277327e. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:45,855 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 456db60834cd049ce89711613277327e that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,856 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/4e209070c1b4810e530b1e749a5db6a1/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,857 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633 and set watcher; region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,857 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=22698c7f211462a832f422e476392633
2013-03-20 16:03:45,857 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:45,858 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:45,858 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 22698c7f211462a832f422e476392633 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,858 INFO [StoreOpenerThread-disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,858 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 108 byte(s) of data from znode /hbase/unassigned/456db60834cd049ce89711613277327e; data=region=enabledTable,,1363820624674.456db60834cd049ce89711613277327e., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,860 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,860 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:45,860 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,861 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/22698c7f211462a832f422e476392633; data=region=enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,862 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(635): Onlined disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.; next sequenceid=1
2013-03-20 16:03:45,862 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node f62bc1775c460d6926da67fdbe4875c2 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,863 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 456db60834cd049ce89711613277327e in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,863 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,863 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.; plan=hri=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,864 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,864 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,} state:enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. state=OFFLINE, ts=1363820625851, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:745)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,865 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,865 INFO [PRI IPC Server handler 7 on 55446] regionserver.HRegionServer(2863): Received request to open region: enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:45,867 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 22698c7f211462a832f422e476392633 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,868 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node f62bc1775c460d6926da67fdbe4875c2 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,868 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,868 INFO [PostOpenDeployTasks:f62bc1775c460d6926da67fdbe4875c2] regionserver.HRegionServer(1729): Post open deploy tasks for region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., daughter=false
2013-03-20 16:03:45,869 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/unassigned/22698c7f211462a832f422e476392633 because node does not exist (not an error)
2013-03-20 16:03:45,871 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,871 DEBUG [PRI IPC Server handler 7 on 55446] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,872 INFO [PostOpenDeployTasks:f62bc1775c460d6926da67fdbe4875c2] catalog.MetaEditor(297): Updated row disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,872 INFO [PostOpenDeployTasks:f62bc1775c460d6926da67fdbe4875c2] regionserver.HRegionServer(1754): Done with post open deploy task for region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., daughter=false
2013-03-20 16:03:45,872 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,872 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node f62bc1775c460d6926da67fdbe4875c2 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,872 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/456db60834cd049ce89711613277327e
2013-03-20 16:03:45,873 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,,1363820624674.456db60834cd049ce89711613277327e. has been deleted.
2013-03-20 16:03:45,873 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,,1363820624674.456db60834cd049ce89711613277327e. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,873 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/22698c7f211462a832f422e476392633
2013-03-20 16:03:45,873 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. has been deleted.
2013-03-20 16:03:45,873 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,aaa,1363820624687.22698c7f211462a832f422e476392633. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,873 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,874 DEBUG [PRI IPC Server handler 7 on 55446] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,876 DEBUG [PRI IPC Server handler 7 on 55446] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,876 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,877 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,877 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}
2013-03-20 16:03:45,877 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 4e7bed8fd93c2959aff778fdb801089c with OFFLINE state
2013-03-20 16:03:45,878 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,878 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(477): Instantiated enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:45,879 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node f62bc1775c460d6926da67fdbe4875c2 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:45,882 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => f62bc1775c460d6926da67fdbe4875c2,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,882 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] handler.OpenRegionHandler(135): Opened disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,882 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,883 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/51bb507664c357c5303e3d230413e1c1/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,884 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,884 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,886 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,887 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,887 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:45,887 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] handler.OpenedRegionHandler(147): Handling OPENED event for disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:45,887 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,887 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for f62bc1775c460d6926da67fdbe4875c2 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,888 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.; plan=hri=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,888 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,888 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,} state:enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. state=OFFLINE, ts=1363820625877, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:745)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,889 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,889 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:45,889 INFO [PRI IPC Server handler 8 on 55446] regionserver.HRegionServer(2863): Received request to open region: enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:45,889 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,889 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,893 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:45,893 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,893 DEBUG [PRI IPC Server handler 8 on 55446] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,893 WARN [Thread-1-EventThread] master.AssignmentManager(916): Received OPENED for region f62bc1775c460d6926da67fdbe4875c2 from server 10.11.3.210,55445,1363820602585 but region was in the state disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. state=OPEN, ts=1363820625877, server=10.11.3.210,55445,1363820602585 and not in expected OFFLINE, PENDING_OPEN or OPENING states
2013-03-20 16:03:45,894 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region f62bc1775c460d6926da67fdbe4875c2 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:45,894 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,897 DEBUG [PRI IPC Server handler 8 on 55446] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,899 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,900 DEBUG [PRI IPC Server handler 8 on 55446] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,902 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}
2013-03-20 16:03:45,902 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,902 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 2900df9d44e0e7dff2a11e46ea809cea with OFFLINE state
2013-03-20 16:03:45,902 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,903 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(477): Instantiated enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:45,903 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,904 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:03:45,904 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. has been deleted.
2013-03-20 16:03:45,904 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:45,904 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:45,905 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/4e7bed8fd93c2959aff778fdb801089c/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,905 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,905 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:45,906 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,908 INFO [StoreOpenerThread-enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,908 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,908 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.; plan=hri=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,908 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,908 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 2900df9d44e0e7dff2a11e46ea809cea,} state:disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. state=OFFLINE, ts=1363820625902, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:750)
2013-03-20 16:03:45,909 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,909 INFO [PRI IPC Server handler 9 on 55446] regionserver.HRegionServer(2863): Received request to open region: disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:45,910 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,911 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,911 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(635): Onlined enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.; next sequenceid=1
2013-03-20 16:03:45,911 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,912 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,914 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,914 DEBUG [PRI IPC Server handler 9 on 55446] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,915 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:45,916 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,916 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., daughter=false
2013-03-20 16:03:45,917 DEBUG [PRI IPC Server handler 9 on 55446] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,917 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,917 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:45,918 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:45,918 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:45,919 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:45,919 DEBUG [PRI IPC Server handler 9 on 55446] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,920 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(4239): Opening region: {NAME => 'disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 2900df9d44e0e7dff2a11e46ea809cea,}
2013-03-20 16:03:45,920 DEBUG [Thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:45,920 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,921 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:45,921 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:45,921 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(477): Instantiated disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:45,922 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/2900df9d44e0e7dff2a11e46ea809cea/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:45,923 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:45,923 INFO [StoreOpenerThread-enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,924 DEBUG [Thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,924 DEBUG [Thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,925 DEBUG [Thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:45,925 INFO [Thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820625920, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.HMaster.assignRegion(HMaster.java:1758)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:750)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:45,925 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,926 INFO [PRI IPC Server handler 0 on 55446] regionserver.HRegionServer(2863): Received request to open region: disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:45,926 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,926 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(635): Onlined enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.; next sequenceid=1
2013-03-20 16:03:45,927 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,927 DEBUG [PRI IPC Server handler 0 on 55446] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,927 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,928 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,929 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,930 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,930 DEBUG [PRI IPC Server handler 0 on 55446] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:45,943 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:45,943 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,943 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., daughter=false
2013-03-20 16:03:45,943 DEBUG [PRI IPC Server handler 0 on 55446] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,945 INFO [Thread-1] master.TestMasterFailover(951):
Waiting for assignment to finish
2013-03-20 16:03:45,946 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,947 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:45,947 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:45,950 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,950 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:45,950 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:45,951 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:45,951 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:45,951 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:45,951 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:45,958 INFO [StoreOpenerThread-disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:45,960 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(635): Onlined disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.; next sequenceid=1
2013-03-20 16:03:45,960 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,961 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,963 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:45,963 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:45,963 INFO [PostOpenDeployTasks:2900df9d44e0e7dff2a11e46ea809cea] regionserver.HRegionServer(1729): Post open deploy tasks for region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., daughter=false
2013-03-20 16:03:45,964 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:45,964 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,054 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,055 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,055 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,055 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,055 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,157 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,158 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,158 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,158 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,158 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,261 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,261 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,261 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,261 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,261 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,277 INFO [StoreOpenerThread-disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:46,280 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.; next sequenceid=1
2013-03-20 16:03:46,280 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e209070c1b4810e530b1e749a5db6a1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,282 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,283 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,283 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e209070c1b4810e530b1e749a5db6a1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,286 INFO [PostOpenDeployTasks:4e209070c1b4810e530b1e749a5db6a1] regionserver.HRegionServer(1729): Post open deploy tasks for region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., daughter=false
2013-03-20 16:03:46,287 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,287 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,289 INFO [PostOpenDeployTasks:4e209070c1b4810e530b1e749a5db6a1] catalog.MetaEditor(297): Updated row disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:46,289 INFO [PostOpenDeployTasks:4e209070c1b4810e530b1e749a5db6a1] regionserver.HRegionServer(1754): Done with post open deploy task for region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., daughter=false
2013-03-20 16:03:46,289 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e209070c1b4810e530b1e749a5db6a1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:46,290 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,292 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,292 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e209070c1b4810e530b1e749a5db6a1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:46,292 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 4e209070c1b4810e530b1e749a5db6a1,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:46,292 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(135): Opened disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:46,293 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:46,293 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,293 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] handler.OpenedRegionHandler(147): Handling OPENED event for disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:46,293 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 4e209070c1b4810e530b1e749a5db6a1 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:46,295 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:46,297 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:03:46,297 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:46,297 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. has been deleted.
2013-03-20 16:03:46,297 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:46,298 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-0] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 4e209070c1b4810e530b1e749a5db6a1 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:46,298 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:46,299 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,300 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,301 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,303 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,364 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,364 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,364 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,364 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,468 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,468 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,468 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,469 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,572 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,572 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,573 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,573 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,583 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,584 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,586 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,586 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,587 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,587 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,610 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,611 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,613 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,613 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,614 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,614 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,630 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,632 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,637 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,637 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:46,638 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:46,638 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,676 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,676 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,676 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,676 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,778 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,778 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,779 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,779 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,882 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,882 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,882 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,882 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:46,985 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:46,985 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:46,985 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:46,985 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,089 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,089 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,089 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,089 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,192 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,192 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,192 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,192 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,253 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,255 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,273 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,273 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,274 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,274 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,280 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,281 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,304 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,308 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,308 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,309 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,310 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,310 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,310 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,310 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,310 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,310 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,337 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,337 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,338 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,338 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,413 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,413 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,413 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,414 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,517 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,517 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,517 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,517 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,621 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,621 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,621 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,622 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,726 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,726 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,726 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,726 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,830 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,830 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,830 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,830 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,934 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,934 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,934 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:47,934 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:47,940 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,941 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,942 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,943 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,943 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,944 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:47,975 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,976 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,977 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:47,978 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:47,978 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:47,979 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,004 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,005 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,007 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,007 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,008 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,008 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,037 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,037 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,037 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,037 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,141 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,141 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,141 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,141 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,245 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,245 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,245 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,245 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,347 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,347 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,348 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,348 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,451 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,452 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,452 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,452 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,555 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,555 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,555 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,556 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,610 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,612 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,613 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,613 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,614 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,614 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,645 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,646 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,648 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,648 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,649 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,649 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,657 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,658 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,658 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,658 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,674 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,676 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,677 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,678 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:48,679 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:48,679 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,761 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,761 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,762 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,762 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,865 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,865 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,865 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,865 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:48,969 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:48,969 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:48,969 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:48,969 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,072 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,072 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,072 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,073 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,175 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,175 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,176 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,176 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,279 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,279 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,279 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,279 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,280 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,282 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,303 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,304 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,305 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,305 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,315 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,316 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,318 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,318 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,319 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,319 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,345 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,347 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,348 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,349 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,350 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,350 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,382 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,382 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,382 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,382 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,485 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,485 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,485 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,485 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,588 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,588 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,588 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,588 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,691 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,692 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,692 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,692 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,795 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,795 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,795 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,795 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,898 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,899 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,899 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:49,899 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:49,970 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,972 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,973 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,974 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,975 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,975 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:49,985 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,986 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,987 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:49,988 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:49,988 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:49,989 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,016 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,017 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,018 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,018 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,019 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,019 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,105 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,106 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,106 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,106 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,213 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,213 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,213 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,213 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,315 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,315 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,315 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,315 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,418 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,419 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,419 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,419 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,514 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3087): Regions in transition timed out: disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OPENING, ts=1363820625930, server=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,514 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager(3135): Region has been OPENING for too long, reassigning region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:50,516 DEBUG [10.11.3.210,55443,1363820596870.timeoutMonitor] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,516 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,518 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,519 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,521 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,521 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,522 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. so generated a random one; hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680; 2 (online=2, available=2) available servers
2013-03-20 16:03:50,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,522 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=PENDING_OPEN, ts=1363820630516, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
2013-03-20 16:03:50,522 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,522 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,523 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,523 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,524 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,524 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=0
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,525 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,527 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,529 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,530 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,531 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,531 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,531 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,531 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,531 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,531 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630525, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,533 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,533 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=1
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,533 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,535 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,536 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,538 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,538 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,538 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,538 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,538 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,538 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,538 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630533, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,540 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,540 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=2
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,540 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,542 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,543 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,544 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,544 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,544 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,545 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,545 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,545 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,545 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630540, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,546 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,547 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=3
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,547 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,549 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,550 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,551 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,551 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,551 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,551 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,551 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,551 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,551 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630547, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,553 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,553 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=4
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,553 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,555 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,557 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,557 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,558 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,558 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,558 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,558 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,558 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,558 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630553, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,559 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,560 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=5
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,560 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,562 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,563 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,564 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,565 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,565 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,565 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,565 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,565 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,565 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630560, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,567 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,567 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=6
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,567 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,569 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,597 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,598 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,599 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,599 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,599 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,599 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,599 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,599 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630567, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,601 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,601 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=7
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,602 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,605 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,606 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,607 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,607 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,607 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,608 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,608 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,608 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,608 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630602, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,610 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,610 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=8
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,611 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:50,613 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,615 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,616 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,616 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,616 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,616 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,616 DEBUG [pool-8-thread-1] master.AssignmentManager(1978): Using pre-existing plan for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; plan=hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,617 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,617 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630611, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:50,618 DEBUG [pool-8-thread-1] master.AssignmentManager(1755): Failed assignment in: 10.11.3.210,55446,1363820602680 due to org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
2013-03-20 16:03:50,618 WARN [pool-8-thread-1] master.AssignmentManager(1769): Failed assignment of disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55446,1363820602680, trying to assign to the same region server because of RegionAlreadyInTransitionException;retry=9
org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:95)
at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:79)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1751)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
Caused by: org.apache.hadoop.ipc.RemoteException: org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException: Received:OPEN for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. for the region:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., which we are already trying to OPEN.
at org.apache.hadoop.hbase.regionserver.HRegionServer.addRegionsInTransition(HRegionServer.java:2951)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2861)
at org.apache.hadoop.hbase.regionserver.HRegionServer.openRegion(HRegionServer.java:2837)
at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:320)
at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1426)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:995)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
at $Proxy19.openRegion(Unknown Source)
at org.apache.hadoop.hbase.master.ServerManager.sendRegionOpen(ServerManager.java:500)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1706)
... 7 more
2013-03-20 16:03:50,625 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,625 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,625 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,626 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,640 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,641 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,642 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,643 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,643 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,644 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,655 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,656 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,661 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,662 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,662 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,663 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,685 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,687 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,688 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,688 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:50,689 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,690 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,728 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,728 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,729 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,729 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,832 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,833 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,833 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,833 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,922 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:03:50,923 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(1597): hconnection Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:03:50,923 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e90008 connected
2013-03-20 16:03:50,924 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90008 Set watcher on existing znode /hbase/master
2013-03-20 16:03:50,925 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90008 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:50,925 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e90008 Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:03:50,926 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90008 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,928 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90008 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,929 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@6ff43d69; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,929 DEBUG [PostOpenDeployTasks:2900df9d44e0e7dff2a11e46ea809cea] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90008 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,930 DEBUG [PostOpenDeployTasks:2900df9d44e0e7dff2a11e46ea809cea] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@6ff43d69; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,930 DEBUG [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90008 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,931 DEBUG [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@6ff43d69; serverName=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,932 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:03:50,939 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,940 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,940 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,940 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:50,941 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] catalog.MetaEditor(297): Updated row enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. with server=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,941 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., daughter=false
2013-03-20 16:03:50,941 INFO [PostOpenDeployTasks:2900df9d44e0e7dff2a11e46ea809cea] catalog.MetaEditor(297): Updated row disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. with server=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,941 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,941 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] catalog.MetaEditor(297): Updated row enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. with server=10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,944 INFO [PostOpenDeployTasks:2900df9d44e0e7dff2a11e46ea809cea] regionserver.HRegionServer(1754): Done with post open deploy task for region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., daughter=false
2013-03-20 16:03:50,944 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., daughter=false
2013-03-20 16:03:50,945 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,946 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,947 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,948 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,948 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:50,949 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,949 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}, server: 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(135): Opened enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. on server:10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 2900df9d44e0e7dff2a11e46ea809cea from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(4239): Opening region: {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,}
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 2900df9d44e0e7dff2a11e46ea809cea,}, server: 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-2] handler.OpenRegionHandler(135): Opened disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. on server:10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,950 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] zookeeper.ZKAssign(820): regionserver:55446-0x13d8a0ac7e90002 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:50,951 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}, server: 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,951 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:50,951 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-1] handler.OpenRegionHandler(135): Opened enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. on server:10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,951 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(477): Instantiated disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:50,951 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,951 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55446,1363820602680, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,951 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,951 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. from 10.11.3.210,55446,1363820602680; deleting unassigned node
2013-03-20 16:03:50,952 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 51bb507664c357c5303e3d230413e1c1 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,952 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea and set watcher; region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,953 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/disabledTable/36716be644bc9c78b3a96e0d4a43c183/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:03:50,953 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55446,1363820602680, region=2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,953 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,953 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,953 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] handler.OpenedRegionHandler(147): Handling OPENED event for disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. from 10.11.3.210,55446,1363820602680; deleting unassigned node
2013-03-20 16:03:50,953 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 2900df9d44e0e7dff2a11e46ea809cea that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,954 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:50,955 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,955 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55446,1363820602680, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,955 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:50,955 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. from 10.11.3.210,55446,1363820602680; deleting unassigned node
2013-03-20 16:03:50,955 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-1] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 51bb507664c357c5303e3d230413e1c1 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,955 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. has been deleted.
2013-03-20 16:03:50,955 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 4e7bed8fd93c2959aff778fdb801089c that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,956 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. that was online on 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,956 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:50,956 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea; data=region=disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,957 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:50,991 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-2] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 2900df9d44e0e7dff2a11e46ea809cea in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,992 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-3] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 4e7bed8fd93c2959aff778fdb801089c in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:50,992 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c because node does not exist (not an error)
2013-03-20 16:03:50,993 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55443-0x13d8a0ac7e90000 Unable to get data of znode /hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea because node does not exist (not an error)
2013-03-20 16:03:50,994 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,994 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:50,994 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. has been deleted.
2013-03-20 16:03:50,994 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea. that was online on 10.11.3.210,55446,1363820602680
2013-03-20 16:03:50,994 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:50,996 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:50,996 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:50,996 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. has been deleted.
2013-03-20 16:03:50,996 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. that was online on 10.11.3.210,55446,1363820602680
2013-03-20 16:03:51,042 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,144 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,246 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,349 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,404 INFO [StoreOpenerThread-disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:51,407 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(635): Onlined disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; next sequenceid=1
2013-03-20 16:03:51,407 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:51,408 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:51,409 WARN [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(777): regionserver:55446-0x13d8a0ac7e90002 Attempt to transition the unassigned node for 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING failed, the node existed but was version 11 not the expected version 1
2013-03-20 16:03:51,409 WARN [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(385): Failed refreshing OPENING; region=36716be644bc9c78b3a96e0d4a43c183, context=post_region_open
2013-03-20 16:03:51,409 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(975): Closing disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.: disabling compactions & flushes
2013-03-20 16:03:51,409 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(997): Updates disabled for region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:51,409 INFO [StoreCloserThread-disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:51,409 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(1045): Closed disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:51,410 INFO [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(301): Opening of region {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} failed, marking as FAILED_OPEN in ZK
2013-03-20 16:03:51,410 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(757): regionserver:55446-0x13d8a0ac7e90002 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_FAILED_OPEN
2013-03-20 16:03:51,411 DEBUG [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKUtil(1597): regionserver:55446-0x13d8a0ac7e90002 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:51,411 WARN [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] zookeeper.ZKAssign(799): regionserver:55446-0x13d8a0ac7e90002 Attempt to transition the unassigned node for 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_FAILED_OPEN failed, the node existed but was in the state M_ZK_REGION_OFFLINE set by the server 10.11.3.210,55446,1363820602680
2013-03-20 16:03:51,411 WARN [RS_OPEN_REGION-10.11.3.210,55446,1363820602680-0] handler.OpenRegionHandler(308): Unable to mark region {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} as FAILED_OPEN. It's likely that the master already timed out this open attempt, and thus another RS already has the region.
2013-03-20 16:03:51,452 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,555 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,657 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,760 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,863 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:51,966 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,069 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,172 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,275 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,378 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,480 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,584 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,687 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,790 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,893 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:52,997 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,100 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,203 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,305 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,408 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,511 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,615 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,718 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,821 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:53,923 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,026 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,128 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,232 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,334 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,437 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,540 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,643 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,745 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,848 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:54,950 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,052 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,155 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,258 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,360 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,463 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,566 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,669 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,771 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,874 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:55,977 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,080 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,183 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,286 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,388 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,491 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,517 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3087): Regions in transition timed out: disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820630619, server=null
2013-03-20 16:03:56,517 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3098): Region has been OFFLINE for too long, reassigning disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to a random server
2013-03-20 16:03:56,517 DEBUG [pool-8-thread-1] zookeeper.ZKAssign(264): master:55443-0x13d8a0ac7e90000 Creating (or updating) unassigned node for 36716be644bc9c78b3a96e0d4a43c183 with OFFLINE state
2013-03-20 16:03:56,519 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:56,521 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,522 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:56,522 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55443,1363820596870, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,522 DEBUG [pool-8-thread-1] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:56,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1953): Found an existing plan for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. destination server is 10.11.3.210,55446,1363820602680
2013-03-20 16:03:56,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. so generated a random one; hri=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., src=, dest=10.11.3.210,55445,1363820602585; 2 (online=2, available=2) available servers
2013-03-20 16:03:56,522 DEBUG [pool-8-thread-1] master.AssignmentManager(1698): Assigning region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. to 10.11.3.210,55445,1363820602585
2013-03-20 16:03:56,523 INFO [pool-8-thread-1] master.AssignmentManager(1701): Current args region:{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,} state:disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. state=OFFLINE, ts=1363820636517, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:03:56,526 INFO [PRI IPC Server handler 6 on 55445] regionserver.HRegionServer(2863): Received request to open region: disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:56,528 DEBUG [PRI IPC Server handler 6 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:56,529 DEBUG [PRI IPC Server handler 6 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55443,1363820596870, state=M_ZK_REGION_OFFLINE
2013-03-20 16:03:56,546 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,547 DEBUG [PRI IPC Server handler 6 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:03:56,548 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(4239): Opening region: {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,}
2013-03-20 16:03:56,549 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:56,549 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:03:56,549 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,549 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(477): Instantiated disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:03:56,553 INFO [StoreOpenerThread-disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:03:56,556 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(635): Onlined disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.; next sequenceid=1
2013-03-20 16:03:56,556 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:56,557 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:56,560 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,560 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:03:56,561 INFO [PostOpenDeployTasks:36716be644bc9c78b3a96e0d4a43c183] regionserver.HRegionServer(1729): Post open deploy tasks for region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., daughter=false
2013-03-20 16:03:56,561 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:56,561 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,565 INFO [PostOpenDeployTasks:36716be644bc9c78b3a96e0d4a43c183] catalog.MetaEditor(297): Updated row disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:03:56,565 INFO [PostOpenDeployTasks:36716be644bc9c78b3a96e0d4a43c183] regionserver.HRegionServer(1754): Done with post open deploy task for region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., daughter=false
2013-03-20 16:03:56,566 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:56,567 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:03:56,572 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,573 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 36716be644bc9c78b3a96e0d4a43c183 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:03:56,573 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:03:56,573 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(135): Opened disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:03:56,573 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:56,573 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,574 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] handler.OpenedRegionHandler(147): Handling OPENED event for disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:03:56,574 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKAssign(483): master:55443-0x13d8a0ac7e90000 Deleting existing unassigned node for 36716be644bc9c78b3a96e0d4a43c183 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:56,575 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:03:56,580 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,581 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:56,581 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. has been deleted.
2013-03-20 16:03:56,581 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:03:56,581 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:03:56,581 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55443,1363820596870-4] zookeeper.ZKAssign(512): master:55443-0x13d8a0ac7e90000 Successfully deleted unassigned node for region 36716be644bc9c78b3a96e0d4a43c183 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:03:56,593 INFO [Thread-1] master.TestMasterFailover(951):
Assignment completed
2013-03-20 16:03:56,593 INFO [Thread-1] master.TestMasterFailover(951):
Aborting master
2013-03-20 16:03:56,593 INFO [Thread-1] hbase.MiniHBaseCluster(410): Aborting org.apache.hadoop.hbase.master.HMaster@6dcbf6bb
2013-03-20 16:03:56,593 FATAL [Thread-1] master.HMaster(1506): Master server abort: loaded coprocessors are: []
2013-03-20 16:03:56,593 FATAL [Thread-1] master.HMaster(1511): Aborting for tests
java.lang.Exception: Trace info
at org.apache.hadoop.hbase.MiniHBaseCluster.abortMaster(MiniHBaseCluster.java:411)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:760)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:03:56,593 INFO [Thread-1] master.HMaster(1670): Aborting
2013-03-20 16:03:56,593 INFO [Thread-1] hbase.LocalHBaseCluster(352): Waiting on 10.11.3.210,55443,1363820596870
2013-03-20 16:03:56,594 INFO [10.11.3.210,55443,1363820596870-BalancerChore] hbase.Chore(80): 10.11.3.210,55443,1363820596870-BalancerChore exiting
2013-03-20 16:03:56,594 DEBUG [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(922): Stopping service threads
2013-03-20 16:03:56,594 INFO [10.11.3.210,55443,1363820596870-CatalogJanitor] hbase.Chore(80): 10.11.3.210,55443,1363820596870-CatalogJanitor exiting
2013-03-20 16:03:56,598 INFO [Master:0;10.11.3.210,55443,1363820596870.oldLogCleaner] hbase.Chore(80): Master:0;10.11.3.210,55443,1363820596870.oldLogCleaner exiting
2013-03-20 16:03:56,598 INFO [Master:0;10.11.3.210,55443,1363820596870.archivedHFileCleaner] hbase.Chore(80): Master:0;10.11.3.210,55443,1363820596870.archivedHFileCleaner exiting
2013-03-20 16:03:56,599 DEBUG [Master:0;10.11.3.210,55443,1363820596870] zookeeper.ZKUtil(1597): master:55443-0x13d8a0ac7e90000 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55443,136...
2013-03-20 16:03:56,601 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55443-0x13d8a0ac7e90000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,601 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,601 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90003 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,601 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90006 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,602 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90004 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,604 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55446-0x13d8a0ac7e90002 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,604 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90008 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2013-03-20 16:03:56,606 DEBUG [Master:0;10.11.3.210,55443,1363820596870] catalog.CatalogTracker(227): Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5de867d6
2013-03-20 16:03:56,606 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90006 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,606 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(425): regionserver:55445-0x13d8a0ac7e90001 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,606 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90004 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,606 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680-EventThread] zookeeper.ZKUtil(425): regionserver:55446-0x13d8a0ac7e90002 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,606 DEBUG [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90008 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,607 DEBUG [Master:0;10.11.3.210,55443,1363820596870-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90003 /hbase/master does not exist. Watcher is set.
2013-03-20 16:03:56,658 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:03:56,658 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:03:56,706 INFO [Master:0;10.11.3.210,55443,1363820596870] client.HConnectionManager$HConnectionImplementation(1785): Closed zookeeper sessionid=0x13d8a0ac7e90003
2013-03-20 16:03:56,711 INFO [10.11.3.210,55443,1363820596870.timeoutMonitor] hbase.Chore(80): 10.11.3.210,55443,1363820596870.timeoutMonitor exiting
2013-03-20 16:03:56,711 INFO [10.11.3.210,55443,1363820596870.splitLogManagerTimeoutMonitor] hbase.Chore(80): 10.11.3.210,55443,1363820596870.splitLogManagerTimeoutMonitor exiting
2013-03-20 16:03:56,711 INFO [10.11.3.210,55443,1363820596870.timerUpdater] hbase.Chore(80): 10.11.3.210,55443,1363820596870.timerUpdater exiting
2013-03-20 16:03:56,720 INFO [Master:0;10.11.3.210,55443,1363820596870] master.HMaster(421): HMaster main thread exiting
2013-03-20 16:03:56,722 INFO [Thread-1] master.TestMasterFailover(951):
Master has aborted
2013-03-20 16:03:56,722 INFO [Thread-1] master.TestMasterFailover(951):
Beginning to mock scenarios
2013-03-20 16:03:56,723 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 7 byte(s) of data from znode /hbase/table/.META.; data=ENABLED
2013-03-20 16:03:56,724 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 7 byte(s) of data from znode /hbase/table/disabledTable; data=ENABLED
2013-03-20 16:03:56,725 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 7 byte(s) of data from znode /hbase/table/enabledTable; data=ENABLED
2013-03-20 16:03:56,726 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 7 byte(s) of data from znode /hbase/table/-ROOT-; data=ENABLED
2013-03-20 16:03:56,751 DEBUG [Thread-1] zookeeper.ZKAssign(561): unittest-0x13d8a0ac7e90007 Creating unassigned node for 51bb507664c357c5303e3d230413e1c1 in a CLOSING state
2013-03-20 16:03:56,758 DEBUG [Thread-1] master.TestMasterFailover(789):
Region of enabled table was CLOSING on dead RS
{NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}
2013-03-20 16:03:56,758 DEBUG [Thread-1] zookeeper.ZKAssign(561): unittest-0x13d8a0ac7e90007 Creating unassigned node for 2900df9d44e0e7dff2a11e46ea809cea in a CLOSING state
2013-03-20 16:03:56,776 DEBUG [Thread-1] master.TestMasterFailover(796):
Region of disabled table was CLOSING on dead RS
{NAME => 'disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 2900df9d44e0e7dff2a11e46ea809cea,}
2013-03-20 16:03:56,777 DEBUG [Thread-1] zookeeper.ZKAssign(561): unittest-0x13d8a0ac7e90007 Creating unassigned node for 4e7bed8fd93c2959aff778fdb801089c in a CLOSING state
2013-03-20 16:03:56,795 DEBUG [Thread-1] zookeeper.ZKAssign(757): unittest-0x13d8a0ac7e90007 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:03:56,797 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=M_ZK_REGION_CLOSING
2013-03-20 16:03:56,799 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:56,799 DEBUG [Thread-1] zookeeper.ZKAssign(820): unittest-0x13d8a0ac7e90007 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:03:56,799 DEBUG [Thread-1] master.TestMasterFailover(808):
Region of enabled table was CLOSED on dead RS
{NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}
2013-03-20 16:03:56,800 DEBUG [Thread-1] zookeeper.ZKAssign(561): unittest-0x13d8a0ac7e90007 Creating unassigned node for 36716be644bc9c78b3a96e0d4a43c183 in a CLOSING state
2013-03-20 16:03:56,814 DEBUG [Thread-1] zookeeper.ZKAssign(757): unittest-0x13d8a0ac7e90007 Attempting to transition node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:03:56,815 DEBUG [Thread-1] zookeeper.ZKUtil(1597): unittest-0x13d8a0ac7e90007 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=M_ZK_REGION_CLOSING
2013-03-20 16:03:56,826 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:03:56,826 DEBUG [Thread-1] zookeeper.ZKAssign(820): unittest-0x13d8a0ac7e90007 Successfully transitioned node 36716be644bc9c78b3a96e0d4a43c183 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:03:56,827 DEBUG [Thread-1] master.TestMasterFailover(816):
Region of disabled table was CLOSED on dead RS
{NAME => 'disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 36716be644bc9c78b3a96e0d4a43c183,}
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(822): To be dead region:70236052
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(822): To be dead region:2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(822): To be dead region:51bb507664c357c5303e3d230413e1c1
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(822): To be dead region:4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(951):
Done mocking data up in ZK
2013-03-20 16:03:56,827 INFO [Thread-1] master.TestMasterFailover(951):
Killing RS 10.11.3.210,55446,1363820602680
2013-03-20 16:03:56,845 FATAL [Thread-1] regionserver.HRegionServer(1787): ABORTING region server 10.11.3.210,55446,1363820602680: Killing for unit test
2013-03-20 16:03:56,845 FATAL [Thread-1] regionserver.HRegionServer(1794): RegionServer abort: loaded coprocessors are: []
2013-03-20 16:03:56,849 INFO [Thread-1] regionserver.HRegionServer(1797): Dump of metrics: requestsPerSecond=0, numberOfOnlineRegions=4, numberOfStores=4, numberOfStorefiles=1, storefileIndexSizeMB=0, rootIndexSizeKB=0, totalStaticIndexSizeKB=0, totalStaticBloomSizeKB=0, memstoreSizeMB=0, mbInMemoryWithoutWAL=0, numberOfPutsWithoutWAL=0, readRequestsCount=45, writeRequestsCount=1, compactionQueueSize=0, flushQueueSize=0, usedHeapMB=72, maxHeapMB=1883, blockCacheSizeMB=3.86, blockCacheFreeMB=466.98, blockCacheCount=1, blockCacheHitCount=25, blockCacheMissCount=1, blockCacheEvictedCount=0, blockCacheHitRatio=96%, blockCacheHitCachingRatio=96%, hdfsBlocksLocalityIndex=0, slowHLogAppendCount=0, fsReadLatencyHistogramMean=0, fsReadLatencyHistogramCount=0, fsReadLatencyHistogramMedian=0, fsReadLatencyHistogram75th=0, fsReadLatencyHistogram95th=0, fsReadLatencyHistogram99th=0, fsReadLatencyHistogram999th=0, fsPreadLatencyHistogramMean=0, fsPreadLatencyHistogramCount=0, fsPreadLatencyHistogramMedian=0, fsPreadLatencyHistogram75th=0, fsPreadLatencyHistogram95th=0, fsPreadLatencyHistogram99th=0, fsPreadLatencyHistogram999th=0, fsWriteLatencyHistogramMean=0, fsWriteLatencyHistogramCount=0, fsWriteLatencyHistogramMedian=0, fsWriteLatencyHistogram75th=0, fsWriteLatencyHistogram95th=0, fsWriteLatencyHistogram99th=0, fsWriteLatencyHistogram999th=0
2013-03-20 16:03:56,850 WARN [Thread-1] regionserver.HRegionServer(1809): Unable to report fatal error to master
java.lang.reflect.UndeclaredThrowableException
at $Proxy18.reportRSFatalError(Unknown Source)
at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:1805)
at org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(MiniHBaseCluster.java:165)
at org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$100(MiniHBaseCluster.java:100)
at org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$2.run(MiniHBaseCluster.java:158)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:337)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1103)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.hbase.util.Methods.call(Methods.java:37)
at org.apache.hadoop.hbase.security.User.call(User.java:590)
at org.apache.hadoop.hbase.security.User.access$700(User.java:51)
at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:430)
at org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.abort(MiniHBaseCluster.java:156)
at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:1818)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:833)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:599)
at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:489)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupConnection(HBaseClient.java:390)
at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:436)
at org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:1124)
at org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:974)
at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:86)
... 28 more
2013-03-20 16:03:56,850 INFO [Thread-1] regionserver.HRegionServer(1710): STOPPED: Killing for unit test
2013-03-20 16:03:56,850 INFO [Thread-1] master.TestMasterFailover(951):
RS 10.11.3.210,55446,1363820602680 killed
2013-03-20 16:03:56,851 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(1938): Connected to master at null
2013-03-20 16:03:56,853 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.SplitLogWorker(522): Sending interrupt to stop the worker thread
2013-03-20 16:03:56,853 INFO [SplitLogWorker-10.11.3.210,55446,1363820602680] regionserver.SplitLogWorker(221): SplitLogWorker interrupted while waiting for task, exiting: java.lang.InterruptedException
2013-03-20 16:03:56,853 INFO [SplitLogWorker-10.11.3.210,55446,1363820602680] regionserver.SplitLogWorker(185): SplitLogWorker 10.11.3.210,55446,1363820602680 exiting
2013-03-20 16:03:56,854 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.cacheFlusher] regionserver.MemStoreFlusher(274): RegionServer:1;10.11.3.210,55446,1363820602680.cacheFlusher exiting
2013-03-20 16:03:56,854 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.compactionChecker] hbase.Chore(80): RegionServer:1;10.11.3.210,55446,1363820602680.compactionChecker exiting
2013-03-20 16:03:56,854 DEBUG [Thread-1.LruBlockCache.EvictionThread] hfile.LruBlockCache(411): Block cache LRU eviction started; Attempting to free -357651.03 KB of total=3.86 MB
2013-03-20 16:03:56,854 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] snapshot.RegionServerSnapshotManager(151): Stopping RegionServerSnapshotManager abruptly.
2013-03-20 16:03:56,854 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.logRoller] regionserver.LogRoller(114): LogRoller exiting.
2013-03-20 16:03:56,855 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-0] handler.CloseRegionHandler(105): Processing close of enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:56,855 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-1] handler.CloseRegionHandler(105): Processing close of disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:56,856 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-2] handler.CloseRegionHandler(105): Processing close of enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:56,857 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(975): Closing disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.: disabling compactions & flushes
2013-03-20 16:03:56,856 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(975): Closing enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.: disabling compactions & flushes
2013-03-20 16:03:56,856 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(826): aborting server 10.11.3.210,55446,1363820602680
2013-03-20 16:03:56,857 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(997): Updates disabled for region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:56,857 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(997): Updates disabled for region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:56,857 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(975): Closing enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.: disabling compactions & flushes
2013-03-20 16:03:56,857 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] catalog.CatalogTracker(227): Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@5b46ac5a
2013-03-20 16:03:56,858 INFO [StoreCloserThread-disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:56,858 INFO [StoreCloserThread-enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:56,857 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(997): Updates disabled for region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:56,858 INFO [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(1045): Closed enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:56,858 INFO [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-1] regionserver.HRegion(1045): Closed disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:56,858 INFO [StoreCloserThread-enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.-1] regionserver.Store(712): Closed family
2013-03-20 16:03:56,858 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-1] handler.CloseRegionHandler(149): Closed region disabledTable,bbb,1363820625619.2900df9d44e0e7dff2a11e46ea809cea.
2013-03-20 16:03:56,858 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-0] handler.CloseRegionHandler(149): Closed region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:03:56,858 INFO [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-2] regionserver.HRegion(1045): Closed enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:56,859 DEBUG [RS_CLOSE_REGION-10.11.3.210,55446,1363820602680-2] handler.CloseRegionHandler(149): Closed region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:03:56,958 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] client.HConnectionManager$HConnectionImplementation(1785): Closed zookeeper sessionid=0x13d8a0ac7e90008
2013-03-20 16:03:56,961 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] snapshot.RegionServerSnapshotManager(151): Stopping RegionServerSnapshotManager abruptly.
2013-03-20 16:03:56,961 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.CompactSplitThread(271): Waiting for Split Thread to finish...
2013-03-20 16:03:56,961 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.CompactSplitThread(271): Waiting for Large Compaction Thread to finish...
2013-03-20 16:03:56,961 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.CompactSplitThread(271): Waiting for Small Compaction Thread to finish...
2013-03-20 16:03:56,962 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(964): Waiting on 1 regions to close
2013-03-20 16:03:56,962 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(968): {70236052=-ROOT-,,0.70236052}
2013-03-20 16:03:56,962 DEBUG [RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680-0] handler.CloseRegionHandler(105): Processing close of -ROOT-,,0.70236052
2013-03-20 16:03:56,962 DEBUG [RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(975): Closing -ROOT-,,0.70236052: disabling compactions & flushes
2013-03-20 16:03:56,962 DEBUG [RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(997): Updates disabled for region -ROOT-,,0.70236052
2013-03-20 16:03:56,964 INFO [StoreCloserThread--ROOT-,,0.70236052-1] regionserver.Store(712): Closed info
2013-03-20 16:03:56,964 INFO [RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680-0] regionserver.HRegion(1045): Closed -ROOT-,,0.70236052
2013-03-20 16:03:56,964 DEBUG [RS_CLOSE_ROOT-10.11.3.210,55446,1363820602680-0] handler.CloseRegionHandler(149): Closed region -ROOT-,,0.70236052
2013-03-20 16:03:57,162 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(856): stopping server 10.11.3.210,55446,1363820602680; all regions closed.
2013-03-20 16:03:57,163 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.logSyncer] wal.HLog$LogSyncer(1255): RegionServer:1;10.11.3.210,55446,1363820602680.logSyncer exiting
2013-03-20 16:03:57,163 DEBUG [RegionServer:1;10.11.3.210,55446,1363820602680] wal.HLog(1017): closing hlog writer in hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680
2013-03-20 16:03:57,221 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.Leases(125): RegionServer:1;10.11.3.210,55446,1363820602680 closing leases
2013-03-20 16:03:57,221 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.Leases(132): RegionServer:1;10.11.3.210,55446,1363820602680 closed leases
2013-03-20 16:03:57,235 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(880): stopping server 10.11.3.210,55446,1363820602680; zookeeper connection closed.
2013-03-20 16:03:57,235 INFO [RegionServer:1;10.11.3.210,55446,1363820602680] regionserver.HRegionServer(883): RegionServer:1;10.11.3.210,55446,1363820602680 exiting
2013-03-20 16:03:57,236 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b4b2db7] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(182): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b4b2db7
2013-03-20 16:03:57,242 INFO [Thread-1] master.TestMasterFailover(951):
Starting up a new master
2013-03-20 16:03:57,297 DEBUG [Thread-1] client.HConnectionManager(1885): Set serverside HConnection retries=100
2013-03-20 16:03:57,300 INFO [Thread-1] ipc.HBaseRpcMetrics(59): Initializing RPC Metrics with hostName=HMaster, port=55539
2013-03-20 16:03:57,302 DEBUG [Thread-1] zookeeper.ZKUtil(120): master:55539 opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:03:57,302 INFO [Thread-1] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:03:57,565 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.leaseChecker] regionserver.Leases(125): RegionServer:1;10.11.3.210,55446,1363820602680.leaseChecker closing leases
2013-03-20 16:03:57,565 INFO [RegionServer:1;10.11.3.210,55446,1363820602680.leaseChecker] regionserver.Leases(132): RegionServer:1;10.11.3.210,55446,1363820602680.leaseChecker closed leases
2013-03-20 16:03:57,659 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:03:58,660 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:03:59,661 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:04:00,661 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:04:01,663 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1899): No master found; retry
2013-03-20 16:04:02,413 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539 Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:04:02,415 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(350): master:55539-0x13d8a0ac7e90009 connected
2013-03-20 16:04:02,424 INFO [Thread-1] metrics.MetricsMBeanBase(160): new MBeanInfo
2013-03-20 16:04:02,424 INFO [Thread-1] metrics.MasterMetrics(89): Initialized
2013-03-20 16:04:02,426 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(423): master:55539-0x13d8a0ac7e90009 Set watcher on existing znode /hbase/shutdown
2013-03-20 16:04:02,427 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 28 byte(s) of data from znode /hbase/shutdown and set watcher; Wed Mar 20 16:03:27 PDT 2013
2013-03-20 16:04:02,429 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2013-03-20 16:04:02,429 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90004 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2013-03-20 16:04:02,429 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90006 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2013-03-20 16:04:02,429 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ActiveMasterManager(143): Deleting ZNode for /hbase/backup-masters/10.11.3.210,55539,1363820637301 from backup master directory
2013-03-20 16:04:02,430 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55539,136...
2013-03-20 16:04:02,431 WARN [Master:0;10.11.3.210,55539,1363820637301] zookeeper.RecoverableZooKeeper(143): Node /hbase/backup-masters/10.11.3.210,55539,1363820637301 already deleted, and this is not a retry
2013-03-20 16:04:02,431 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ActiveMasterManager(150): Master=10.11.3.210,55539,1363820637301
2013-03-20 16:04:02,431 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55539,136...
2013-03-20 16:04:02,431 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55539,136...
2013-03-20 16:04:02,435 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(175): timeout = 25000
2013-03-20 16:04:02,435 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(176): unassigned timeout = 180000
2013-03-20 16:04:02,436 INFO [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(833): found 0 orphan tasks and 0 rescan nodes
2013-03-20 16:04:02,467 DEBUG [Master:0;10.11.3.210,55539,1363820637301] catalog.CatalogTracker(210): Starting catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@4f8f104
2013-03-20 16:04:02,468 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(423): master:55539-0x13d8a0ac7e90009 Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:04:02,468 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:04:02,469 INFO [Thread-1] master.ServerManager(377): Master doesn't enable ServerShutdownHandler during initialization, delay expiring server 10.11.3.210,55446,1363820602680
2013-03-20 16:04:02,469 INFO [Thread-1] master.TestMasterFailover(951):
Waiting for master to be ready
2013-03-20 16:04:02,469 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(425): master:55539-0x13d8a0ac7e90009 /hbase/unassigned/1028785192 does not exist. Watcher is set.
2013-03-20 16:04:02,471 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 7 byte(s) of data from znode /hbase/table/.META.; data=ENABLED
2013-03-20 16:04:02,472 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 9 byte(s) of data from znode /hbase/table/disabledTable; data=DISABLING
2013-03-20 16:04:02,472 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 7 byte(s) of data from znode /hbase/table/enabledTable; data=ENABLED
2013-03-20 16:04:02,473 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 7 byte(s) of data from znode /hbase/table/-ROOT-; data=ENABLED
2013-03-20 16:04:02,474 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(423): master:55539-0x13d8a0ac7e90009 Set watcher on existing znode /hbase/rs/10.11.3.210,55445,1363820602585
2013-03-20 16:04:02,475 INFO [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(476): Server active/primary master; 10.11.3.210,55539,1363820637301, sessionid=0x13d8a0ac7e90009, cluster-up flag was=true
2013-03-20 16:04:02,475 INFO [Master:0;10.11.3.210,55539,1363820637301] snapshot.SnapshotManager(903): Snapshot feature is not enabled, missing log and hfile cleaners.
2013-03-20 16:04:02,477 INFO [Master:0;10.11.3.210,55539,1363820637301] zookeeper.RecoverableZooKeeper(440): Node /hbase/online-snapshot/acquired already exists and this is not a retry
2013-03-20 16:04:02,478 INFO [Master:0;10.11.3.210,55539,1363820637301] procedure.ZKProcedureUtil(266): Clearing all procedure znodes: /hbase/online-snapshot/acquired /hbase/online-snapshot/reached /hbase/online-snapshot/abort
2013-03-20 16:04:02,480 DEBUG [Master:0;10.11.3.210,55539,1363820637301] procedure.ZKProcedureCoordinatorRpcs(194): Starting the controller for procedure member:10.11.3.210,55539,1363820637301
2013-03-20 16:04:02,480 WARN [Master:0;10.11.3.210,55539,1363820637301] snapshot.SnapshotManager(234): Couldn't delete working snapshot directory: hdfs://localhost:55430/user/jzhong/hbase/.snapshot/.tmp
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] executor.ExecutorService(192): Starting executor service name=MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301, corePoolSize=5, maxPoolSize=5
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] executor.ExecutorService(192): Starting executor service name=MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301, corePoolSize=5, maxPoolSize=5
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] executor.ExecutorService(192): Starting executor service name=MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301, corePoolSize=3, maxPoolSize=3
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] executor.ExecutorService(192): Starting executor service name=MASTER_META_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301, corePoolSize=5, maxPoolSize=5
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] executor.ExecutorService(192): Starting executor service name=MASTER_TABLE_OPERATIONS-10.11.3.210,55539,1363820637301, corePoolSize=1, maxPoolSize=1
2013-03-20 16:04:02,481 DEBUG [Master:0;10.11.3.210,55539,1363820637301] cleaner.CleanerChore(86): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2013-03-20 16:04:02,482 DEBUG [Master:0;10.11.3.210,55539,1363820637301] cleaner.CleanerChore(86): initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2013-03-20 16:04:02,482 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(915): Started service threads
2013-03-20 16:04:02,482 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(629): Waiting for region servers count to settle; currently checked in 0, slept for 0 ms, expecting minimum of 1, maximum of 2, timeout of 4500 ms, interval of 1500 ms.
2013-03-20 16:04:02,663 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1909): Attempting connect to Master server at 10.11.3.210,55539,1363820637301
2013-03-20 16:04:02,667 INFO [RegionServer:0;10.11.3.210,55445,1363820602585] regionserver.HRegionServer(1938): Connected to master at /10.11.3.210:55539
2013-03-20 16:04:03,669 INFO [IPC Server handler 1 on 55539] master.ServerManager(274): Registering server=10.11.3.210,55445,1363820602585
2013-03-20 16:04:03,704 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(629): Waiting for region servers count to settle; currently checked in 1, slept for 1222 ms, expecting minimum of 1, maximum of 2, timeout of 4500 ms, interval of 1500 ms.
2013-03-20 16:04:05,228 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(629): Waiting for region servers count to settle; currently checked in 1, slept for 2746 ms, expecting minimum of 1, maximum of 2, timeout of 4500 ms, interval of 1500 ms.
2013-03-20 16:04:06,754 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(629): Waiting for region servers count to settle; currently checked in 1, slept for 4272 ms, expecting minimum of 1, maximum of 2, timeout of 4500 ms, interval of 1500 ms.
2013-03-20 16:04:07,009 INFO [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(646): Finished waiting for region servers count to settle; checked in 1, slept for 4527 ms, expecting minimum of 1, maximum of 2, master is running.
2013-03-20 16:04:07,011 INFO [Master:0;10.11.3.210,55539,1363820637301] master.MasterFileSystem(237): Log folder hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585 belongs to an existing region server
2013-03-20 16:04:07,012 INFO [Master:0;10.11.3.210,55539,1363820637301] master.MasterFileSystem(233): Log folder hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680 doesn't belong to a known region server, splitting
2013-03-20 16:04:07,032 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.MasterFileSystem(283): Renamed region directory: hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting
2013-03-20 16:04:07,032 INFO [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(935): dead splitlog workers [10.11.3.210,55446,1363820602680]
2013-03-20 16:04:07,033 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(249): Scheduling batch of logs to split
2013-03-20 16:04:07,048 INFO [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(251): started splitting logs in [hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting]
2013-03-20 16:04:07,060 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/splitlog
2013-03-20 16:04:07,061 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] regionserver.SplitLogWorker(509): tasks arrived or departed
2013-03-20 16:04:07,062 DEBUG [Thread-1-EventThread] master.SplitLogManager(442): put up splitlog task at znode /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:07,062 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 42 byte(s) of data from znode /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488; data=unassigned 10.11.3.210,55539,...
2013-03-20 16:04:07,063 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] regionserver.SplitLogWorker(273): worker 10.11.3.210,55445,1363820602585 acquired task /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:07,064 INFO [Thread-1-EventThread] master.SplitLogManager(560): task /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488 acquired by 10.11.3.210,55445,1363820602585
2013-03-20 16:04:07,072 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.HLogSplitter(377): Splitting hlog: hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting/10.11.3.210%2C55446%2C1363820602680.1363820608488, length=308
2013-03-20 16:04:07,098 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] util.FSHDFSUtils(66): Recovering file hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting/10.11.3.210%2C55446%2C1363820602680.1363820608488
2013-03-20 16:04:07,440 DEBUG [10.11.3.210,55539,1363820637301.splitLogManagerTimeoutMonitor] master.SplitLogManager$TimeoutMonitor(989): total tasks = 1 unassigned = 0
2013-03-20 16:04:08,100 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] util.FSHDFSUtils(120): Finished lease recover attempt for hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting/10.11.3.210%2C55446%2C1363820602680.1363820608488
2013-03-20 16:04:08,299 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,299 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,341 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.SequenceFileLogWriter(193): using new createWriter -- HADOOP-6840
2013-03-20 16:04:08,341 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.SequenceFileLogWriter(204): Path=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004.temp, syncFs=true, hflush=false, compression=false
2013-03-20 16:04:08,342 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.HLogSplitter(1065): Creating writer path=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004.temp region=70236052
2013-03-20 16:04:08,414 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.HLogSplitter(472): Closed hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004.temp
2013-03-20 16:04:08,421 DEBUG [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.HLogSplitter(491): Rename hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004.temp to hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004
2013-03-20 16:04:08,421 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] wal.HLogSplitter(498): Processed 1 edits across 1 regions threw away edits for 0 regions; log file=hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting/10.11.3.210%2C55446%2C1363820602680.1363820608488 is corrupted = false progress failed = false
2013-03-20 16:04:08,440 DEBUG [10.11.3.210,55539,1363820637301.splitLogManagerTimeoutMonitor] master.SplitLogManager$TimeoutMonitor(989): total tasks = 1 unassigned = 0
2013-03-20 16:04:08,462 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,462 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,462 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] regionserver.SplitLogWorker(388): successfully transitioned task /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488 to final state done
2013-03-20 16:04:08,462 INFO [SplitLogWorker-10.11.3.210,55445,1363820602585] regionserver.SplitLogWorker(322): worker 10.11.3.210,55445,1363820602585 done with task /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488 in 1398ms
2013-03-20 16:04:08,463 INFO [Thread-1-EventThread] master.SplitLogManager(492): task /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488 entered state done 10.11.3.210,55445,1363820602585
2013-03-20 16:04:08,468 DEBUG [Thread-1-EventThread] wal.HLogSplitter(602): Archived processed log hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting/10.11.3.210%2C55446%2C1363820602680.1363820608488 to hdfs://localhost:55430/user/jzhong/hbase/.oldlogs/10.11.3.210%2C55446%2C1363820602680.1363820608488
2013-03-20 16:04:08,479 INFO [Thread-1-EventThread] master.SplitLogManager(404): Done splitting /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,481 INFO [Master:0;10.11.3.210,55539,1363820637301] master.SplitLogManager(299): finished splitting (more than or equal to) 308 bytes in 1 log files in [hdfs://localhost:55430/user/jzhong/hbase/.logs/10.11.3.210,55446,1363820602680-splitting] in 1432ms
2013-03-20 16:04:08,481 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:08,482 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/splitlog
2013-03-20 16:04:08,482 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] regionserver.SplitLogWorker(509): tasks arrived or departed
2013-03-20 16:04:08,482 DEBUG [Thread-1-EventThread] master.SplitLogManager$DeleteAsyncCallback(1159): deleted /hbase/splitlog/hdfs%3A%2F%2Flocalhost%3A55430%2Fuser%2Fjzhong%2Fhbase%2F.logs%2F10.11.3.210%2C55446%2C1363820602680-splitting%2F10.11.3.210%252C55446%252C1363820602680.1363820608488
2013-03-20 16:04:10,484 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/70236052 because node does not exist (not an error)
2013-03-20 16:04:10,486 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:04:10,487 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(120): hconnection opening connection to ZooKeeper with ensemble (localhost:54650)
2013-03-20 16:04:10,488 INFO [Master:0;10.11.3.210,55539,1363820637301] zookeeper.RecoverableZooKeeper(104): The identifier of this process is 38592@JZhongs-MacBook-Pro.local
2013-03-20 16:04:10,490 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2013-03-20 16:04:10,492 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZooKeeperWatcher(350): hconnection-0x13d8a0ac7e9000a connected
2013-03-20 16:04:10,492 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=86367a84-7905-477f-9b59-b1976...
2013-03-20 16:04:10,493 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e9000a Set watcher on existing znode /hbase/master
2013-03-20 16:04:10,494 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 33 byte(s) of data from znode /hbase/master and set watcher; \x00\x0010.11.3.210,55539,136...
2013-03-20 16:04:10,495 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(423): hconnection-0x13d8a0ac7e9000a Set watcher on existing znode /hbase/root-region-server
2013-03-20 16:04:10,496 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:04:10,498 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55446,1363820602680
2013-03-20 16:04:10,498 INFO [Master:0;10.11.3.210,55539,1363820637301] catalog.RootLocationEditor(43): Unsetting ROOT region location in ZooKeeper
2013-03-20 16:04:10,499 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e9000a Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,499 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,500 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90004 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,500 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90006 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,500 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,502 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKAssign(264): master:55539-0x13d8a0ac7e90009 Creating (or updating) unassigned node for 70236052 with OFFLINE state
2013-03-20 16:04:10,502 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90004 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:04:10,502 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(425): regionserver:55445-0x13d8a0ac7e90001 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:04:10,502 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e90006 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:04:10,503 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZKUtil(425): hconnection-0x13d8a0ac7e9000a /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:04:10,503 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(425): master:55539-0x13d8a0ac7e90009 /hbase/root-region-server does not exist. Watcher is set.
2013-03-20 16:04:10,506 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(1993): Removing dead but not expired server: 10.11.3.210,55446,1363820602680 from eligible server pool.
2013-03-20 16:04:10,506 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for -ROOT-,,0.70236052 so generated a random one; hri=-ROOT-,,0.70236052, src=, dest=10.11.3.210,55445,1363820602585; 1 (online=1, available=1) available servers
2013-03-20 16:04:10,506 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(1698): Assigning region -ROOT-,,0.70236052 to 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,506 INFO [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(1701): Current args region:{NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,} state:-ROOT-,,0.70236052 state=OFFLINE, ts=1363820650502, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.AssignmentManager.assignRoot(AssignmentManager.java:2291)
at org.apache.hadoop.hbase.master.HMaster.assignRootAndMeta(HMaster.java:678)
at org.apache.hadoop.hbase.master.HMaster.finishInitialization(HMaster.java:575)
at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:386)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:10,506 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(564): New connection to 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,510 INFO [PRI IPC Server handler 9 on 55445] regionserver.HRegionServer(2863): Received request to open region: -ROOT-,,0.70236052
2013-03-20 16:04:10,510 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,511 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:10,512 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:04:10,512 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 70236052 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,513 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}
2013-03-20 16:04:10,513 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:04:10,513 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:10,513 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated -ROOT-,,0.70236052
2013-03-20 16:04:10,513 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=70236052/-ROOT-
2013-03-20 16:04:10,516 INFO [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(214): time to purge deletes set to 0ms in store info
2013-03-20 16:04:10,536 DEBUG [StoreOpenerThread--ROOT-,,0.70236052-1] regionserver.Store(448): loaded hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/0336aa79cf35408694eeb2efd751df86, isReference=false, isBulkLoadResult=false, seqid=2, majorCompaction=false
2013-03-20 16:04:10,549 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(3091): Replaying edits from hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004; minSequenceid=2; path=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004
2013-03-20 16:04:10,552 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,553 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:10,554 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:04:10,554 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,555 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(3225): Applied 2, skipped 0, firstSequenceidInLog=4, maxSequenceidInLog=4, path=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004
2013-03-20 16:04:10,555 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1499): Started memstore flush for -ROOT-,,0.70236052, current region memstore size 384.0; wal is null, using passed sequenceid=4
2013-03-20 16:04:10,555 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1548): Finished snapshotting -ROOT-,,0.70236052, commencing wait for mvcc, flushsize=384
2013-03-20 16:04:10,555 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1558): Finished snapshotting, commencing flushing stores
2013-03-20 16:04:10,555 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:10,555 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=70236052/-ROOT-
2013-03-20 16:04:10,558 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/8572a0d054864c7aa793b5fd367b19c3 with permission=rwxrwxrwx
2013-03-20 16:04:10,559 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] hfile.HFileWriterV2(148): Initialized with CacheConfig:enabled [cacheDataOnRead=true] [cacheDataOnWrite=false] [cacheIndexesOnWrite=false] [cacheBloomsOnWrite=false] [cacheEvictOnClose=false] [cacheCompressed=false]
2013-03-20 16:04:10,559 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.StoreFile$Writer(1047): Delete Family Bloom filter type for hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/8572a0d054864c7aa793b5fd367b19c3: CompoundBloomFilterWriter
2013-03-20 16:04:10,968 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.StoreFile$Writer(1267): NO General Bloom and NO DeleteFamily was added to HFile (hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/8572a0d054864c7aa793b5fd367b19c3)
2013-03-20 16:04:10,968 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.Store(843): Flushed , sequenceid=4, memsize=384.0, into tmp file hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/8572a0d054864c7aa793b5fd367b19c3
2013-03-20 16:04:10,977 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.Store(868): Renaming flushed file at hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/.tmp/8572a0d054864c7aa793b5fd367b19c3 to hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/8572a0d054864c7aa793b5fd367b19c3
2013-03-20 16:04:10,987 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.Store(891): Added hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/info/8572a0d054864c7aa793b5fd367b19c3, entries=2, sequenceid=4, filesize=805.0
2013-03-20 16:04:10,988 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1632): Finished memstore flush of ~384.0/384, currentsize=0.0/0 for region -ROOT-,,0.70236052 in 433ms, sequenceid=4, compaction requested=false; wal=null
2013-03-20 16:04:10,989 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(3071): Deleted recovered.edits file=hdfs://localhost:55430/user/jzhong/hbase/-ROOT-/70236052/recovered.edits/0000000000000000004
2013-03-20 16:04:10,990 INFO [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined -ROOT-,,0.70236052; next sequenceid=5
2013-03-20 16:04:10,990 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,991 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:10,993 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:04:10,993 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:10,993 INFO [PostOpenDeployTasks:70236052] regionserver.HRegionServer(1729): Post open deploy tasks for region=-ROOT-,,0.70236052, daughter=false
2013-03-20 16:04:10,993 INFO [PostOpenDeployTasks:70236052] catalog.RootLocationEditor(62): Setting ROOT region location in ZooKeeper as 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,993 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:10,994 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=70236052/-ROOT-
2013-03-20 16:04:10,994 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e9000a Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,994 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90006 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,994 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,995 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZooKeeperWatcher(273): hconnection-0x13d8a0ac7e90004 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,995 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZooKeeperWatcher(273): regionserver:55445-0x13d8a0ac7e90001 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/root-region-server
2013-03-20 16:04:10,996 DEBUG [Master:0;10.11.3.210,55539,1363820637301-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,996 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,997 DEBUG [RegionServer:0;10.11.3.210,55445,1363820602585-EventThread] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,997 DEBUG [PostOpenDeployTasks:1028785192-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90004 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,998 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e90006 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:10,998 INFO [PostOpenDeployTasks:70236052] regionserver.HRegionServer(1754): Done with post open deploy task for region=-ROOT-,,0.70236052, daughter=false
2013-03-20 16:04:10,999 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 70236052/-ROOT- from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:10,999 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,040 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:04:11,040 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 70236052 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:11,040 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,040 DEBUG [RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(135): Opened -ROOT-,,0.70236052 on server:10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,041 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052 and set watcher; region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,041 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=70236052/-ROOT-
2013-03-20 16:04:11,041 INFO [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-0] handler.OpenedRegionHandler(145): Handling OPENED event for -ROOT-,,0.70236052 from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:04:11,042 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 70236052 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,042 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 56 byte(s) of data from znode /hbase/unassigned/70236052; data=region=-ROOT-,,0, origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,060 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/70236052
2013-03-20 16:04:11,060 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region -ROOT-,,0.70236052 has been deleted.
2013-03-20 16:04:11,060 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 70236052 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,060 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region -ROOT-,,0.70236052 that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,061 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,061 INFO [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(692): -ROOT- assigned=1, rit=false, location=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,062 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/1028785192 because node does not exist (not an error)
2013-03-20 16:04:11,063 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,064 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@7d8aecf1; serverName=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,065 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,065 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@7d8aecf1; serverName=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,071 DEBUG [Master:0;10.11.3.210,55539,1363820637301] catalog.CatalogTracker(483): Set new cached META location: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,071 INFO [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(719): .META. assigned=1, rit=false, location=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,072 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/70236052 because node does not exist (not an error)
2013-03-20 16:04:11,072 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(3215): based on AM, current region=-ROOT-,,0.70236052 is on server=10.11.3.210,55445,1363820602585 server being checked: 10.11.3.210,55446,1363820602680
2013-03-20 16:04:11,073 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/1028785192 because node does not exist (not an error)
2013-03-20 16:04:11,073 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(3215): based on AM, current region=.META.,,1.1028785192 is on server=10.11.3.210,55445,1363820602585 server being checked: 10.11.3.210,55446,1363820602680
2013-03-20 16:04:11,074 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.ServerManager(421): Added=10.11.3.210,55446,1363820602680 to dead servers, submitted shutdown handler to be executed, root=false, meta=false
2013-03-20 16:04:11,074 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] handler.ServerShutdownHandler(184): Splitting logs for 10.11.3.210,55446,1363820602680
2013-03-20 16:04:11,075 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.MasterFileSystem(285): Log dir for server 10.11.3.210,55446,1363820602680 does not exist
2013-03-20 16:04:11,075 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.MasterFileSystem(292): No logs to split
2013-03-20 16:04:11,075 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,077 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@7d8aecf1; serverName=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,077 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(90): Creating scanner over -ROOT- starting at key ''
2013-03-20 16:04:11,078 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:04:11,080 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,080 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@7d8aecf1; serverName=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,081 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): hconnection-0x13d8a0ac7e9000a Retrieved 31 byte(s) of data from znode /hbase/root-region-server and set watcher; 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,081 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] client.HConnectionManager$HConnectionImplementation(875): Looked up root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@7d8aecf1; serverName=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,085 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] client.HConnectionManager$HConnectionImplementation(1266): Cached location for .META.,,1.1028785192 is 10.11.3.210:55445
2013-03-20 16:04:11,085 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:04:11,085 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:04:11,086 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(185): Finished with scanning at {NAME => '-ROOT-,,0', STARTKEY => '', ENDKEY => '', ENCODED => 70236052,}
2013-03-20 16:04:11,086 INFO [Master:0;10.11.3.210,55539,1363820637301] catalog.MetaMigrationRemovingHTD(224): Meta version=0; migrated=true
2013-03-20 16:04:11,086 INFO [Master:0;10.11.3.210,55539,1363820637301] catalog.MetaMigrationRemovingHTD(245): ROOT/Meta already up-to date with new HRI.
2013-03-20 16:04:11,086 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:04:11,087 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:04:11,090 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:04:11,091 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:04:11,092 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] handler.ServerShutdownHandler(456): The table disabledTable is disabled. Hence not assigning region2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:04:11,092 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 1d9bcd12253b489d89684f8633af577a has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,092 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region b2b4d44172d9b4b915784938ffa5b554 has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,092 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region a823f0c9e9f93d1c1869aa0694d909d5 has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,093 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 7be9f84106d3cb4303831b1aed868843 has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,093 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 1e926241c785805f2e85cd7fe855c59f has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,093 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 0446dbe5de2b1fc0f6dc5fe104d3c9bb has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,093 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 1e3107abecaed7fa08e7f8b23772f051 has null regionLocation. But its table disabledTable isn't in ENABLING state.
2013-03-20 16:04:11,093 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 595e6cadb10b50694b52a5e6725b4de6 has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(264): master:55539-0x13d8a0ac7e90009 Creating (or updating) unassigned node for 51bb507664c357c5303e3d230413e1c1 with OFFLINE state
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 2a4e0f9d90178bbfecf09633a6933dfa has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 2ddca7a4038739a14492abc07a636d93 has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region e90cd915ccae23af895b832a4dbc17d7 has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 3383249d8093141c26266486bef59f3c has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region d2428d5f096ddcafeb05949a84963f6d has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,094 WARN [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2624): Region 828029ef2e0b58f76fbea0de565e9274 has null regionLocation. But its table enabledTable isn't in ENABLING state.
2013-03-20 16:04:11,095 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(434): Found {NAME => 'disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.', STARTKEY => '', ENDKEY => 'aaa', ENCODED => f62bc1775c460d6926da67fdbe4875c2,}=10.11.3.210,55445,1363820602585 out on cluster
2013-03-20 16:04:11,095 INFO [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(462): Found regions out on cluster or in RIT; failover
2013-03-20 16:04:11,096 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55446,1363820602680, state=M_ZK_REGION_CLOSING
2013-03-20 16:04:11,096 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183 and set watcher; region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,097 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,098 INFO [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(535): Processing region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183. in state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,098 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,098 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. so generated a random one; hri=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., src=, dest=10.11.3.210,55445,1363820602585; 1 (online=1, available=1) available servers
2013-03-20 16:04:11,098 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1698): Assigning region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. to 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,099 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,} state:enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. state=OFFLINE, ts=1363820651094, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.handler.ServerShutdownHandler.process(ServerShutdownHandler.java:298)
at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:171)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:11,099 INFO [PRI IPC Server handler 5 on 55445] regionserver.HRegionServer(2863): Received request to open region: enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:04:11,100 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,101 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,109 DEBUG [Master:0;10.11.3.210,55539,1363820637301] handler.ClosedRegionHandler(92): Handling CLOSED event for 36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:04:11,110 DEBUG [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(1356): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183.
2013-03-20 16:04:11,110 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,110 DEBUG [PRI IPC Server handler 5 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,110 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 36716be644bc9c78b3a96e0d4a43c183 that is in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,110 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}
2013-03-20 16:04:11,111 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:04:11,111 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,111 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(477): Instantiated enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.
2013-03-20 16:04:11,112 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183; data=region=disabledTable,ccc,1363820625621.36716be644bc9c78b3a96e0d4a43c183., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,114 DEBUG [Master:0;10.11.3.210,55539,1363820637301] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 36716be644bc9c78b3a96e0d4a43c183 in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,114 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,114 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:04:11,114 INFO [StoreOpenerThread-enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:04:11,114 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,114 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(264): master:55539-0x13d8a0ac7e90009 Creating (or updating) unassigned node for 4e7bed8fd93c2959aff778fdb801089c with OFFLINE state
2013-03-20 16:04:11,114 INFO [Master:0;10.11.3.210,55539,1363820637301] master.AssignmentManager(2733): The table disabledTable is in DISABLING state. Hence recovering by moving the table to DISABLED state.
2013-03-20 16:04:11,115 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(90): Creating scanner over .META. starting at key 'disabledTable,,'
2013-03-20 16:04:11,115 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(198): Advancing internal scanner to startKey at 'disabledTable,,'
2013-03-20 16:04:11,117 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] regionserver.HRegion(635): Onlined enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.; next sequenceid=1
2013-03-20 16:04:11,117 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,117 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,121 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55446,1363820602680, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,121 INFO [Master:0;10.11.3.210,55539,1363820637301] handler.DisableTableHandler(97): Attemping to disable table disabledTable
2013-03-20 16:04:11,121 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,121 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,122 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,123 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,123 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., daughter=false
2013-03-20 16:04:11,124 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,124 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55539,1363820637301, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,126 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,126 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,129 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,129 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,129 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] catalog.MetaEditor(297): Updated row enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,129 INFO [PostOpenDeployTasks:51bb507664c357c5303e3d230413e1c1] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., daughter=false
2013-03-20 16:04:11,130 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:11,130 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. so generated a random one; hri=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., src=, dest=10.11.3.210,55445,1363820602585; 1 (online=1, available=1) available servers
2013-03-20 16:04:11,130 DEBUG [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1698): Assigning region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. to 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,131 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,} state:enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. state=OFFLINE, ts=1363820651114, server=null setOfflineInZK:true forceNewPlan:false
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1399)
2013-03-20 16:04:11,131 INFO [Master:0;10.11.3.210,55539,1363820637301] handler.DisableTableHandler(121): Offlining 2 regions.
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1394)
at org.apache.hadoop.hbase.master.handler.ServerShutdownHandler.process(ServerShutdownHandler.java:298)
at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:171)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:11,131 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,131 INFO [PRI IPC Server handler 9 on 55445] regionserver.HRegionServer(2863): Received request to open region: enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:04:11,133 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,149 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,149 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 51bb507664c357c5303e3d230413e1c1 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:11,149 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1.', STARTKEY => 'bbb', ENDKEY => 'ccc', ENCODED => 51bb507664c357c5303e3d230413e1c1,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,150 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2] handler.OpenRegionHandler(135): Opened enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,150 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2064): Starting unassignment of region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. (offlining)
2013-03-20 16:04:11,150 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-1] master.AssignmentManager(2064): Starting unassignment of region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. (offlining)
2013-03-20 16:04:11,150 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] zookeeper.ZKAssign(561): master:55539-0x13d8a0ac7e90009 Creating unassigned node for f62bc1775c460d6926da67fdbe4875c2 in a CLOSING state
2013-03-20 16:04:11,151 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,151 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 and set watcher; region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,152 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,152 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}
2013-03-20 16:04:11,152 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] handler.ServerShutdownHandler(300): 2 regions which were planned to open on 10.11.3.210,55446,1363820602680 have been re-assigned.
2013-03-20 16:04:11,153 INFO [MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0] handler.ServerShutdownHandler(305): Finished processing of shutdown of 10.11.3.210,55446,1363820602680
2013-03-20 16:04:11,153 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:04:11,153 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(477): Instantiated enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.
2013-03-20 16:04:11,153 INFO [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.ServerManager(539): Close region request is sent for region:f62bc1775c460d6926da67fdbe4875c2
java.lang.Exception
at org.apache.hadoop.hbase.master.ServerManager.sendRegionClose(ServerManager.java:540)
at org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:2161)
at org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:2045)
at org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler$1.run(DisableTableHandler.java:158)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:11,154 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=51bb507664c357c5303e3d230413e1c1
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:11,154 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-1] zookeeper.ZKAssign(561): master:55539-0x13d8a0ac7e90009 Creating unassigned node for 4e209070c1b4810e530b1e749a5db6a1 in a CLOSING state
2013-03-20 16:04:11,154 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,154 INFO [PRI IPC Server handler 2 on 55445] regionserver.HRegionServer(2999): Received close region: disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.. Version of ZK closing node:0
2013-03-20 16:04:11,155 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] handler.CloseRegionHandler(105): Processing close of disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,155 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2163): Sent CLOSE to 10.11.3.210,55445,1363820602585 for region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,155 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(975): Closing disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.: disabling compactions & flushes
2013-03-20 16:04:11,156 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(997): Updates disabled for region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,156 INFO [StoreCloserThread-disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.-1] regionserver.Store(712): Closed family
2013-03-20 16:04:11,156 INFO [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(1045): Closed disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,156 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,156 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node f62bc1775c460d6926da67fdbe4875c2 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,157 INFO [StoreOpenerThread-enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:04:11,157 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,157 INFO [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-1] master.ServerManager(539): Close region request is sent for region:4e209070c1b4810e530b1e749a5db6a1
java.lang.Exception
at org.apache.hadoop.hbase.master.ServerManager.sendRegionClose(ServerManager.java:540)
at org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:2161)
at org.apache.hadoop.hbase.master.AssignmentManager.unassign(AssignmentManager.java:2045)
2013-03-20 16:04:11,157 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
at org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler$1.run(DisableTableHandler.java:158)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:11,159 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-1] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:04:11,159 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 51bb507664c357c5303e3d230413e1c1 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,160 INFO [PRI IPC Server handler 1 on 55445] regionserver.HRegionServer(2999): Received close region: disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.. Version of ZK closing node:0
2013-03-20 16:04:11,160 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_CLOSING
2013-03-20 16:04:11,162 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] handler.CloseRegionHandler(105): Processing close of disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,162 DEBUG [10.11.3.210,55539,1363820637301-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-1] master.AssignmentManager(2163): Sent CLOSE to 10.11.3.210,55445,1363820602585 for region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,164 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(975): Closing disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.: disabling compactions & flushes
2013-03-20 16:04:11,164 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(997): Updates disabled for region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,166 INFO [StoreCloserThread-disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.-1] regionserver.Store(712): Closed family
2013-03-20 16:04:11,166 INFO [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(1045): Closed disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,166 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e209070c1b4810e530b1e749a5db6a1 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,166 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node f62bc1775c460d6926da67fdbe4875c2 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,166 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] handler.CloseRegionHandler(140): set region closed state in zk successfully for region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2. sn name: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,166 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0] handler.CloseRegionHandler(149): Closed region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,166 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] regionserver.HRegion(635): Onlined enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.; next sequenceid=1
2013-03-20 16:04:11,167 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,167 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1; data=region=enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,168 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_CLOSING
2013-03-20 16:04:11,168 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,169 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 51bb507664c357c5303e3d230413e1c1 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,169 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,170 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_CLOSING
2013-03-20 16:04:11,170 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e209070c1b4810e530b1e749a5db6a1 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,171 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] handler.CloseRegionHandler(140): set region closed state in zk successfully for region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1. sn name: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,171 DEBUG [RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1] handler.CloseRegionHandler(149): Closed region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,171 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:11,171 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/51bb507664c357c5303e3d230413e1c1 because node does not exist (not an error)
2013-03-20 16:04:11,171 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., daughter=false
2013-03-20 16:04:11,172 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,172 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:04:11,173 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 and set watcher; region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,173 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_CLOSED, server=10.11.3.210,55445,1363820602585, region=f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:04:11,173 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/51bb507664c357c5303e3d230413e1c1
2013-03-20 16:04:11,174 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0] handler.ClosedRegionHandler(92): Handling CLOSED event for f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:04:11,174 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. has been deleted.
2013-03-20 16:04:11,175 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0] master.AssignmentManager(1356): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2.
2013-03-20 16:04:11,175 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for f62bc1775c460d6926da67fdbe4875c2 that is in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,175 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,bbb,1363820624690.51bb507664c357c5303e3d230413e1c1. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,175 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] catalog.MetaEditor(297): Updated row enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,176 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,176 INFO [PostOpenDeployTasks:4e7bed8fd93c2959aff778fdb801089c] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., daughter=false
2013-03-20 16:04:11,176 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:11,177 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 109 byte(s) of data from znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2; data=region=disabledTable,,1363820625613.f62bc1775c460d6926da67fdbe4875c2., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,178 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:11,179 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 4e7bed8fd93c2959aff778fdb801089c from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:11,179 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region f62bc1775c460d6926da67fdbe4875c2 in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,179 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c.', STARTKEY => 'ccc', ENDKEY => 'ddd', ENCODED => 4e7bed8fd93c2959aff778fdb801089c,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,179 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0] handler.OpenRegionHandler(135): Opened enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,180 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,180 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,181 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2 because node does not exist (not an error)
2013-03-20 16:04:11,181 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:04:11,182 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 and set watcher; region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,182 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_CLOSED, server=10.11.3.210,55445,1363820602585, region=4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:04:11,182 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,183 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1] handler.ClosedRegionHandler(92): Handling CLOSED event for 4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:04:11,183 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1] master.AssignmentManager(1356): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.
2013-03-20 16:04:11,183 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 4e209070c1b4810e530b1e749a5db6a1 that is in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,183 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,184 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,184 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/f62bc1775c460d6926da67fdbe4875c2
2013-03-20 16:04:11,184 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,184 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-2] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:04:11,184 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-2] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 4e7bed8fd93c2959aff778fdb801089c that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,184 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1; data=region=disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,186 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-2] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c; data=region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,201 DEBUG [Master:0;10.11.3.210,55539,1363820637301] handler.DisableTableHandler$BulkDisabler(173): Disable waiting until done; 300000 ms remaining; [{NAME => 'disabledTable,aaa,1363820625617.4e209070c1b4810e530b1e749a5db6a1.', STARTKEY => 'aaa', ENDKEY => 'bbb', ENCODED => 4e209070c1b4810e530b1e749a5db6a1,}]
2013-03-20 16:04:11,223 DEBUG [MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 4e209070c1b4810e530b1e749a5db6a1 in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:11,224 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c and set watcher; region=enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:11,224 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-2] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 4e7bed8fd93c2959aff778fdb801089c in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:11,225 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(713): master:55539-0x13d8a0ac7e90009 Unable to get data of znode /hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1 because node does not exist (not an error)
2013-03-20 16:04:11,225 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/4e209070c1b4810e530b1e749a5db6a1
2013-03-20 16:04:11,225 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,225 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/4e7bed8fd93c2959aff778fdb801089c
2013-03-20 16:04:11,226 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. has been deleted.
2013-03-20 16:04:11,226 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,ccc,1363820624692.4e7bed8fd93c2959aff778fdb801089c. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:04:11,252 DEBUG [Master:0;10.11.3.210,55539,1363820637301] handler.DisableTableHandler$BulkDisabler(173): Disable waiting until done; 299949 ms remaining; []
2013-03-20 16:04:11,256 INFO [Master:0;10.11.3.210,55539,1363820637301] handler.DisableTableHandler(137): Disabled table is done=true
2013-03-20 16:04:11,257 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:04:11,257 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:04:11,261 DEBUG [Master:0;10.11.3.210,55539,1363820637301] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:04:11,262 INFO [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(1955): Registered HMaster MXBean
2013-03-20 16:04:11,262 DEBUG [10.11.3.210,55539,1363820637301-CatalogJanitor] client.ClientScanner(90): Creating scanner over .META. starting at key ''
2013-03-20 16:04:11,262 DEBUG [10.11.3.210,55539,1363820637301-CatalogJanitor] client.ClientScanner(198): Advancing internal scanner to startKey at ''
2013-03-20 16:04:11,262 INFO [Master:0;10.11.3.210,55539,1363820637301] master.HMaster(607): Master has completed initialization
2013-03-20 16:04:11,267 DEBUG [10.11.3.210,55539,1363820637301-CatalogJanitor] client.ClientScanner(185): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,}
2013-03-20 16:04:11,267 DEBUG [10.11.3.210,55539,1363820637301-CatalogJanitor] master.CatalogJanitor(156): Scanned 22 catalog row(s) and gc'd 0 unreferenced parent region(s)
2013-03-20 16:04:11,344 INFO [Thread-1] master.TestMasterFailover(951):
Master is ready
2013-03-20 16:04:11,344 DEBUG [Thread-1] zookeeper.ZKAssign(142): unittest-0x13d8a0ac7e90007 Creating unassigned node for 595e6cadb10b50694b52a5e6725b4de6 in OFFLINE state
2013-03-20 16:04:11,346 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,347 DEBUG [Thread-1] zookeeper.ZKAssign(142): unittest-0x13d8a0ac7e90007 Creating unassigned node for 1d9bcd12253b489d89684f8633af577a in OFFLINE state
2013-03-20 16:04:11,349 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,350 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:11,350 INFO [Thread-1] master.TestMasterFailover(951):
Waiting for no more RIT
2013-03-20 16:04:11,352 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,352 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,355 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,356 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a and set watcher; region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:11,459 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,459 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,561 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,561 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,664 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,664 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,766 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,766 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,869 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,869 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:11,972 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:11,972 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,076 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,076 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,178 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,178 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,281 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,281 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,384 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,384 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,487 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,487 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,589 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,590 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,692 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,692 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,795 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,796 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:12,899 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:12,899 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,002 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,013 INFO [10.11.3.210,55539,1363820637301.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3087): Regions in transition timed out: disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a. state=PENDING_OPEN, ts=0, server=null
2013-03-20 16:04:13,013 INFO [10.11.3.210,55539,1363820637301.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3103): Region has been PENDING_OPEN for too long, reassigning region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a.
2013-03-20 16:04:13,014 INFO [10.11.3.210,55539,1363820637301.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3087): Regions in transition timed out: enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. state=PENDING_OPEN, ts=0, server=null
2013-03-20 16:04:13,014 INFO [10.11.3.210,55539,1363820637301.timeoutMonitor] master.AssignmentManager$TimeoutMonitor(3103): Region has been PENDING_OPEN for too long, reassigning region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.
2013-03-20 16:04:13,015 DEBUG [pool-44-thread-1] zookeeper.ZKAssign(264): master:55539-0x13d8a0ac7e90009 Creating (or updating) unassigned node for 1d9bcd12253b489d89684f8633af577a with OFFLINE state
2013-03-20 16:04:13,015 DEBUG [pool-44-thread-2] zookeeper.ZKAssign(264): master:55539-0x13d8a0ac7e90009 Creating (or updating) unassigned node for 595e6cadb10b50694b52a5e6725b4de6 with OFFLINE state
2013-03-20 16:04:13,018 DEBUG [pool-44-thread-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a; data=region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,018 DEBUG [pool-44-thread-2] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6; data=region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,028 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,028 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,029 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,030 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a and set watcher; region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,030 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55539,1363820637301, region=1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,030 DEBUG [pool-44-thread-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a and set watcher; region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,030 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,030 INFO [pool-44-thread-1] master.AssignmentManager(1803): Table disabledTable disabled; skipping assign of disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a.
2013-03-20 16:04:13,031 DEBUG [pool-44-thread-1] master.AssignmentManager(1356): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a.
2013-03-20 16:04:13,031 DEBUG [pool-44-thread-1] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 1d9bcd12253b489d89684f8633af577a that is in expected state RS_ZK_REGION_CLOSED
2013-03-20 16:04:13,031 DEBUG [pool-44-thread-2] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,031 DEBUG [pool-44-thread-2] master.AssignmentManager(1970): No previous transition plan was found (or we are ignoring an existing plan) for enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. so generated a random one; hri=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., src=, dest=10.11.3.210,55445,1363820602585; 1 (online=1, available=1) available servers
2013-03-20 16:04:13,031 DEBUG [pool-44-thread-2] master.AssignmentManager(1698): Assigning region enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. to 10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,031 INFO [pool-44-thread-2] master.AssignmentManager(1701): Current args region:{NAME => 'enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 595e6cadb10b50694b52a5e6725b4de6,} state:enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. state=PENDING_OPEN, ts=1363820653015, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1703)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
2013-03-20 16:04:13,031 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,032 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=M_ZK_REGION_OFFLINE, server=10.11.3.210,55539,1363820637301, region=595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,032 INFO [PRI IPC Server handler 9 on 55445] regionserver.HRegionServer(2863): Received request to open region: enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.
2013-03-20 16:04:13,032 DEBUG [pool-44-thread-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a; data=region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,033 WARN [pool-44-thread-1] zookeeper.ZKAssign(495): master:55539-0x13d8a0ac7e90009 Attempting to delete unassigned node 1d9bcd12253b489d89684f8633af577a in RS_ZK_REGION_CLOSED state but node is in M_ZK_REGION_OFFLINE state
2013-03-20 16:04:13,033 DEBUG [pool-44-thread-1] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 1d9bcd12253b489d89684f8633af577a that is in expected state M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,034 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 595e6cadb10b50694b52a5e6725b4de6 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:13,034 DEBUG [pool-44-thread-1] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 112 byte(s) of data from znode /hbase/unassigned/1d9bcd12253b489d89684f8633af577a; data=region=disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,044 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/1d9bcd12253b489d89684f8633af577a
2013-03-20 16:04:13,044 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:13,044 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a. has been deleted.
2013-03-20 16:04:13,044 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:13,045 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6; data=region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55539,1363820637301, state=M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,045 DEBUG [pool-44-thread-1] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 1d9bcd12253b489d89684f8633af577a in expected state M_ZK_REGION_OFFLINE
2013-03-20 16:04:13,045 INFO [pool-44-thread-1] master.AssignmentManager(1663): Skipped assignement for disabled table!!! Current args region:{NAME => 'disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 1d9bcd12253b489d89684f8633af577a,} state:disabledTable,ddd,1363820625625.1d9bcd12253b489d89684f8633af577a. state=PENDING_OPEN, ts=1363820653015, server=null setOfflineInZK:true forceNewPlan:true
java.lang.Exception
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1666)
at org.apache.hadoop.hbase.master.AssignmentManager.assign(AssignmentManager.java:1424)
at org.apache.hadoop.hbase.master.AssignCallable.call(AssignCallable.java:44)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,049 DEBUG [PRI IPC Server handler 9 on 55445] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 595e6cadb10b50694b52a5e6725b4de6 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING
2013-03-20 16:04:13,050 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(4239): Opening region: {NAME => 'enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 595e6cadb10b50694b52a5e6725b4de6,}
2013-03-20 16:04:13,050 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:13,050 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,050 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(483): Setting up tabledescriptor config now ...
2013-03-20 16:04:13,050 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(477): Instantiated enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.
2013-03-20 16:04:13,051 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:13,051 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,052 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] util.FSUtils(156): Creating file=hdfs://localhost:55430/user/jzhong/hbase/enabledTable/595e6cadb10b50694b52a5e6725b4de6/.tmp/.regioninfo with permission=rwxrwxrwx
2013-03-20 16:04:13,104 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,208 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,311 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,414 DEBUG [Thread-1] zookeeper.ZKAssign(954): ZK RIT -> 595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,484 INFO [StoreOpenerThread-enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.-1] regionserver.Store(214): time to purge deletes set to 0ms in store family
2013-03-20 16:04:13,486 INFO [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] regionserver.HRegion(635): Onlined enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.; next sequenceid=1
2013-03-20 16:04:13,486 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 595e6cadb10b50694b52a5e6725b4de6 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:13,488 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6; data=region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:13,489 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,489 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 595e6cadb10b50694b52a5e6725b4de6 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING
2013-03-20 16:04:13,490 INFO [PostOpenDeployTasks:595e6cadb10b50694b52a5e6725b4de6] regionserver.HRegionServer(1729): Post open deploy tasks for region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., daughter=false
2013-03-20 16:04:13,490 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:13,490 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENING, server=10.11.3.210,55445,1363820602585, region=595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,492 INFO [PostOpenDeployTasks:595e6cadb10b50694b52a5e6725b4de6] catalog.MetaEditor(297): Updated row enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. with server=10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,492 INFO [PostOpenDeployTasks:595e6cadb10b50694b52a5e6725b4de6] regionserver.HRegionServer(1754): Done with post open deploy task for region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., daughter=false
2013-03-20 16:04:13,493 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(757): regionserver:55445-0x13d8a0ac7e90001 Attempting to transition node 595e6cadb10b50694b52a5e6725b4de6 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:13,494 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKUtil(1597): regionserver:55445-0x13d8a0ac7e90001 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6; data=region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENING
2013-03-20 16:04:13,495 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,496 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] zookeeper.ZKAssign(820): regionserver:55445-0x13d8a0ac7e90001 Successfully transitioned node 595e6cadb10b50694b52a5e6725b4de6 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED
2013-03-20 16:04:13,496 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(281): region transitioned to opened in zookeeper: {NAME => 'enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6.', STARTKEY => 'ddd', ENDKEY => 'eee', ENCODED => 595e6cadb10b50694b52a5e6725b4de6,}, server: 10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,496 DEBUG [RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1] handler.OpenRegionHandler(135): Opened enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. on server:10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,496 DEBUG [Thread-1-EventThread] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6 and set watcher; region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:13,497 DEBUG [Thread-1-EventThread] master.AssignmentManager(744): Handling transition=RS_ZK_REGION_OPENED, server=10.11.3.210,55445,1363820602585, region=595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,497 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-3] handler.OpenedRegionHandler(147): Handling OPENED event for enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. from 10.11.3.210,55445,1363820602585; deleting unassigned node
2013-03-20 16:04:13,497 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-3] zookeeper.ZKAssign(483): master:55539-0x13d8a0ac7e90009 Deleting existing unassigned node for 595e6cadb10b50694b52a5e6725b4de6 that is in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:13,499 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-3] zookeeper.ZKUtil(1597): master:55539-0x13d8a0ac7e90009 Retrieved 111 byte(s) of data from znode /hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6; data=region=enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6., origin=10.11.3.210,55445,1363820602585, state=RS_ZK_REGION_OPENED
2013-03-20 16:04:13,500 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/595e6cadb10b50694b52a5e6725b4de6
2013-03-20 16:04:13,500 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): unittest-0x13d8a0ac7e90007 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:13,500 DEBUG [Thread-1-EventThread] master.AssignmentManager(1177): The znode of region enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. has been deleted.
2013-03-20 16:04:13,500 INFO [Thread-1-EventThread] master.AssignmentManager(1189): The master has opened the region enabledTable,ddd,1363820624694.595e6cadb10b50694b52a5e6725b4de6. that was online on 10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,500 DEBUG [Thread-1-EventThread] zookeeper.ZooKeeperWatcher(273): master:55539-0x13d8a0ac7e90009 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned
2013-03-20 16:04:13,501 DEBUG [MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-3] zookeeper.ZKAssign(512): master:55539-0x13d8a0ac7e90009 Successfully deleted unassigned node for region 595e6cadb10b50694b52a5e6725b4de6 in expected state RS_ZK_REGION_OPENED
2013-03-20 16:04:13,516 INFO [Thread-1] master.TestMasterFailover(951):
No more RIT in ZK
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(951):
No more RIT in RIT map, doing final test verification
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(951):
Current Live Region Server:10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(951):
Current Live Region Server:10.11.3.210,55446,1363820602680
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(913): Got RegionServerStoppedException
org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 10.11.3.210,55446,1363820602680 not running, aborting
at org.apache.hadoop.hbase.regionserver.HRegionServer.checkOpen(HRegionServer.java:3394)
at org.apache.hadoop.hbase.regionserver.HRegionServer.getOnlineRegions(HRegionServer.java:3208)
at org.apache.hadoop.hbase.master.TestMasterFailover.testMasterFailoverWithMockedRITOnDeadRS(TestMasterFailover.java:911)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
at org.junit.internal.runners.statements.FailOnTimeout$StatementThread.run(FailOnTimeout.java:62)
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(951):
Expected offline region:2900df9d44e0e7dff2a11e46ea809cea
2013-03-20 16:04:13,517 INFO [Thread-1] master.TestMasterFailover(951):
Expected offline region:36716be644bc9c78b3a96e0d4a43c183
2013-03-20 16:04:13,523 INFO [main] hbase.ResourceChecker(157): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: 167 threads (was 4), 302 file descriptors (was 94). 3 connections, -thread leak?- -file handle leak?-
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - SessionTracker
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataXceiver@14c3dd7e
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,525 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): java.io.BufferedInputStream.read(BufferedInputStream.java:317)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): java.io.DataInputStream.read(DataInputStream.java:132)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.readToBuf(BlockReceiver.java:265)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.readNextPacket(BlockReceiver.java:312)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:376)
2013-03-20 16:04:13,526 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:532)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:398)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:107)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 0 on 55445
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,527 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Timer-1
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.mainLoop(Timer.java:509)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.run(Timer.java:462)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 4 on 55539
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,528 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585.logRoller
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 8 on 55430
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,529 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 2 on 55439
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-44-thread-2
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,530 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-3-thread-1
2013-03-20 16:04:13,531 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:333)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,532 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 0 on 55439
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 8 on 55445
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,533 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 0 on port 55445
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,534 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 6 on port 55539
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,535 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-8-thread-1
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,536 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@2658dd2d
2013-03-20 16:04:13,537 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,537 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:368)
2013-03-20 16:04:13,537 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,537 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 0 on 55445
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 3 on port 55445
2013-03-20 16:04:13,539 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server listener on 55539
2013-03-20 16:04:13,540 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:636)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to 10.11.3.210/10.11.3.210:55445 from jzhong.hfs.0
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:534)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:579)
2013-03-20 16:04:13,541 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 5 on 55430
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server Responder
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,542 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Responder.run(Server.java:605)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585.compactionChecker
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 0 on port 55539
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,543 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - refreshUsed-/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data2
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.fs.DU$DURefreshThread.run(DU.java:80)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,544 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - DataStreamer for file /user/jzhong/hbase/.logs/10.11.3.210,55445,1363820602585/10.11.3.210%2C55445%2C1363820602585.1363820608489 block blk_-5865301607756890704_1013
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2806)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 3 on 55445
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,545 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Master:0;10.11.3.210,55539,1363820637301
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.master.HMaster.loop(HMaster.java:489)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:387)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor@63f8247d
2013-03-20 16:04:13,546 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.namenode.DecommissionManager$Monitor.run(DecommissionManager.java:65)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - SyncThread:0
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,547 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:97)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 0 on 55445
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,548 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 3 on port 55539
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,549 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 3 on 55445
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,550 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to /10.11.3.210:55445 from jzhong.hfs.2
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:534)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:579)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@69b0de2e
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:150)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:84)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:131)
2013-03-20 16:04:13,551 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-0
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - DataNode: [/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data3,/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data4]
2013-03-20 16:04:13,552 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataNode.offerService(DataNode.java:1023)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataNode.run(DataNode.java:1458)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585.cacheFlusher
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:225)
2013-03-20 16:04:13,553 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - LeaseChecker
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1379)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-3
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,554 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 4 on port 55445
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,555 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-0
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,556 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingReplicationMonitor@2afb6c5f
2013-03-20 16:04:13,557 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.namenode.PendingReplicationBlocks$PendingReplicationMonitor.run(PendingReplicationBlocks.java:186)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - DataNode: [/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data1,/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data2]
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataNode.offerService(DataNode.java:1023)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataNode.run(DataNode.java:1458)
2013-03-20 16:04:13,558 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 0 on 55539
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 457929895@qtp-513470641-0
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,559 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 2057009018@qtp-976393586-0
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 1256033805@qtp-1580650567-0
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-0
2013-03-20 16:04:13,560 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 6 on 55430
2013-03-20 16:04:13,561 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor@45389b3f
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.namenode.FSNamesystem$ReplicationMonitor.run(FSNamesystem.java:2716)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,562 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 1 on 55539
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 1 on 55430
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,563 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 2 on port 55539
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,564 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PostOpenDeployTasks:1028785192-SendThread(localhost:54650)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,565 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 0 on 55436
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,566 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 7 on port 55445
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,567 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - LeaseChecker
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,568 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1379)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PacketResponder 0 for Block blk_-5865301607756890704_1013
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Object.java:485)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:794)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,569 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585-SendThread(localhost:54650)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 1 on 55436
2013-03-20 16:04:13,570 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-2
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,571 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server Responder
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Responder.run(Server.java:605)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 3 on 55430
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,572 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PostOpenDeployTasks:1028785192-EventThread
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,573 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,576 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,576 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 1 on port 55539
2013-03-20 16:04:13,576 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,577 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 5 on port 55445
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,578 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@135d4273
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:150)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:84)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:131)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 1165535969@qtp-1580650567-1 - Acceptor0 SelectChannelConnector@localhost:55431
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,579 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-SendThread(fe80:0:0:0:0:0:0:1%1:54650)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,580 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Timer-0
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.mainLoop(Timer.java:509)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.run(Timer.java:462)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 6 on port 55445
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,581 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-44-thread-1
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
2013-03-20 16:04:13,582 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Async Block Report Generator
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.FSDataset$AsyncBlockReport.waitForReportRequest(FSDataset.java:2254)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.FSDataset$AsyncBlockReport.run(FSDataset.java:2224)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585.logSyncer
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,583 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1240)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_OPEN_REGION-10.11.3.210,55445,1363820602585-0
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585-EventThread
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,584 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - ResponseProcessor for block blk_-5865301607756890704_1013
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
2013-03-20 16:04:13,585 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): java.io.DataInputStream.readFully(DataInputStream.java:178)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): java.io.DataInputStream.readLong(DataInputStream.java:399)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:124)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2967)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Master:0;10.11.3.210,55539,1363820637301.archivedHFileCleaner
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,586 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Master:0;10.11.3.210,55539,1363820637301-SendThread(localhost:54650)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,587 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 9 on 55430
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server listener on 55439
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,588 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener.run(Server.java:439)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 898220330@qtp-976393586-1 - Acceptor0 SelectChannelConnector@localhost:55438
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
2013-03-20 16:04:13,589 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 1 on 55439
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,590 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-EventThread
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 10.11.3.210,55539,1363820637301.timeoutMonitor
2013-03-20 16:04:13,591 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - SplitLogWorker-10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Object.java:485)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:219)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:179)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,592 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - ProcessThread(sid:0 cport:-1):
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:120)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 1 on port 55445
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,593 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 8 on port 55445
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,594 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-SendThread(localhost:54650)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,595 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_SERVER_OPERATIONS-10.11.3.210,55539,1363820637301-0
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,596 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 0 on 55539
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_CLOSE_REGION-10.11.3.210,55445,1363820602585-1
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,597 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 1 on 55445
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,598 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 9 on 55445
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 5 on 55445
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,599 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 10.11.3.210,55539,1363820637301.splitLogManagerTimeoutMonitor
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataXceiver@34374ed5
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,600 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): java.io.BufferedInputStream.read(BufferedInputStream.java:317)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): java.io.DataInputStream.read(DataInputStream.java:132)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.readToBuf(BlockReceiver.java:265)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.readNextPacket(BlockReceiver.java:312)
2013-03-20 16:04:13,601 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:376)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:532)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:398)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:107)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-SendThread(localhost:54650)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,602 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 1 on 55539
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 3 on 55539
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,603 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PacketResponder 1 for Block blk_-5865301607756890704_1013
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Object.java:485)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:794)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 2 on 55539
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,604 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Master:0;10.11.3.210,55539,1363820637301-EventThread
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 7 on port 55539
2013-03-20 16:04:13,605 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,606 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,606 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,606 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,608 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor@1451ef4e
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.namenode.FSNamesystem$HeartbeatMonitor.run(FSNamesystem.java:2693)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 9 on port 55539
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,609 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Timer-2
2013-03-20 16:04:13,610 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.mainLoop(Timer.java:509)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): java.util.TimerThread.run(Timer.java:462)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataBlockScanner@40537935
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:620)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 4 on 55445
2013-03-20 16:04:13,611 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 1 on 55445
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,612 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to localhost/127.0.0.1:55430 from jzhong.hfs.0
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:706)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.run(Client.java:748)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server listener on 55436
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener.run(Server.java:439)
2013-03-20 16:04:13,613 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 2 on 55445
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 7 on 55445
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,614 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server Responder
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:825)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:808)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 2 on 55436
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,615 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 7 on 55430
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-1-thread-1
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,616 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:333)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to localhost/127.0.0.1:55430 from jzhong
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:706)
2013-03-20 16:04:13,617 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.run(Client.java:748)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 6 on 55445
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server listener on 55430
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,618 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener.run(Server.java:439)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server listener on 55445
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:636)
2013-03-20 16:04:13,619 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_OPEN_ROOT-10.11.3.210,55445,1363820602585-0
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 2 on 55430
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,620 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 4 on port 55539
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,621 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - refreshUsed-/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data4
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.fs.DU$DURefreshThread.run(DU.java:80)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_OPEN_REGION-10.11.3.210,55445,1363820602585-2
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,622 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - pool-2-thread-1
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:333)
2013-03-20 16:04:13,623 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:772)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(MiniHBaseCluster.java:147)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$000(MiniHBaseCluster.java:100)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$1.run(MiniHBaseCluster.java:131)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): java.security.AccessController.doPrivileged(Native Method)
2013-03-20 16:04:13,624 INFO [main] hbase.ResourceChecker(185): javax.security.auth.Subject.doAs(Subject.java:337)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1103)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): java.lang.reflect.Method.invoke(Method.java:597)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Methods.call(Methods.java:37)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.security.User.call(User.java:590)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.security.User.access$700(User.java:51)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:430)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.run(MiniHBaseCluster.java:129)
2013-03-20 16:04:13,625 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-EventThread
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_CLOSE_REGION-10.11.3.210,55539,1363820637301-1
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,626 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 2 on 55539
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RegionServer:0;10.11.3.210,55445,1363820602585.leaseChecker
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,627 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:196)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 2 on 55445
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,628 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 8 on port 55539
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,629 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 4 on 55445
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - refreshUsed-/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data3
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.fs.DU$DURefreshThread.run(DU.java:80)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Async Block Report Generator
2013-03-20 16:04:13,630 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.FSDataset$AsyncBlockReport.waitForReportRequest(FSDataset.java:2254)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.FSDataset$AsyncBlockReport.run(FSDataset.java:2224)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - org.apache.hadoop.hdfs.server.datanode.DataBlockScanner@1aa632be
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hdfs.server.datanode.DataBlockScanner.run(DataBlockScanner.java:620)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server Responder
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,631 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Responder.run(Server.java:605)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 9 on port 55445
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,632 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 5 on port 55539
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,633 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Thread-1-EventThread
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Reader 2 on port 55445
2013-03-20 16:04:13,634 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:528)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:514)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - NIOServerCxn.Factory:0.0.0.0/0.0.0.0:54650
2013-03-20 16:04:13,635 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 10.11.3.210,55539,1363820637301.timerUpdater
2013-03-20 16:04:13,636 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 10.11.3.210,55539,1363820637301-CatalogJanitor
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,638 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_OPEN_REGION-10.11.3.210,55445,1363820602585-1
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - MASTER_OPEN_REGION-10.11.3.210,55539,1363820637301-1
2013-03-20 16:04:13,639 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 4 on 55430
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,640 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 10.11.3.210,55539,1363820637301-BalancerChore
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - PRI IPC Server handler 1 on 55445
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,641 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server handler 0 on 55430
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Server$Handler.run(Server.java:1364)
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Server Responder
2013-03-20 16:04:13,642 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:825)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:808)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - REPL IPC Server handler 2 on 55445
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,643 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1398)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - refreshUsed-/Users/jzhong/work/hbase0.94/hbase/target/test-data/848f96fe-94b8-4d1c-8bc7-90601172b290/dfscluster_7e9fb49e-bb9e-49d2-bd72-7f960d623a85/dfs/data/data1
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.sleep(Native Method)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.fs.DU$DURefreshThread.run(DU.java:80)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - 1101513860@qtp-513470641-1 - Acceptor0 SelectChannelConnector@localhost:55435
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
2013-03-20 16:04:13,644 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:136)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:69)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
2013-03-20 16:04:13,645 INFO [main] hbase.ResourceChecker(185): org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(185): org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to localhost/127.0.0.1:55430 from jzhong.hfs.2
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:706)
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.ipc.Client$Connection.run(Client.java:748)
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - Master:0;10.11.3.210,55539,1363820637301.oldLogCleaner
2013-03-20 16:04:13,646 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.Chore.run(Chore.java:75)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - IPC Client (47) connection to /10.11.3.210:55539 from jzhong.hfs.0
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): java.lang.Object.wait(Native Method)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:534)
2013-03-20 16:04:13,647 INFO [main] hbase.ResourceChecker(185): org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:579)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(182): after master.TestMasterFailover#testMasterFailoverWithMockedRITOnDeadRS: potentially hanging thread - RS_OPEN_META-10.11.3.210,55445,1363820602585-0
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): sun.misc.Unsafe.park(Native Method)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.LockSupport.park(LockSupport.java:156)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
2013-03-20 16:04:13,648 INFO [main] hbase.ResourceChecker(185): java.lang.Thread.run(Thread.java:680)