org.apache.hadoop.hbase.coprocessor.TestMasterObserver.testTableOperations Failing for the past 1 build (Since Unstable#29 ) Took 9.5 sec. add description Error Message org.apache.hadoop.hbase.InvalidFamilyOperationException: Column family 'fam2' does not exist at org.apache.hadoop.hbase.master.handler.TableEventHandler.hasColumnFamily(TableEventHandler.java:182) at org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler.(TableDeleteFamilyHandler.java:42) at org.apache.hadoop.hbase.master.HMaster.deleteColumn(HMaster.java:1184) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1344) Stacktrace org.apache.hadoop.hbase.InvalidFamilyOperationException: org.apache.hadoop.hbase.InvalidFamilyOperationException: Column family 'fam2' does not exist at org.apache.hadoop.hbase.master.handler.TableEventHandler.hasColumnFamily(TableEventHandler.java:182) at org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler.(TableDeleteFamilyHandler.java:42) at org.apache.hadoop.hbase.master.HMaster.deleteColumn(HMaster.java:1184) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1344) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27) at java.lang.reflect.Constructor.newInstance(Constructor.java:513) at org.apache.hadoop.hbase.RemoteExceptionHandler.decodeRemoteException(RemoteExceptionHandler.java:96) at org.apache.hadoop.hbase.client.HBaseAdmin.deleteColumn(HBaseAdmin.java:984) at org.apache.hadoop.hbase.coprocessor.TestMasterObserver.testTableOperations(TestMasterObserver.java:621) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:30) at org.junit.runners.ParentRunner.run(ParentRunner.java:300) at org.apache.maven.surefire.junit4.JUnit4TestSet.execute(JUnit4TestSet.java:53) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:123) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:104) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray(ReflectionUtils.java:164) at org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:110) at org.apache.maven.surefire.booter.SurefireStarter.invokeProvider(SurefireStarter.java:175) at org.apache.maven.surefire.booter.SurefireStarter.runSuitesInProcessWhenForked(SurefireStarter.java:81) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:68) Standard Error 2013-08-10 15:19:38,860 DEBUG [main] zookeeper.ZKUtil(96): hconnection opening connection to ZooKeeper with ensemble (localhost:21818) 2013-08-10 15:19:38,862 INFO [main] zookeeper.RecoverableZooKeeper(90): The identifier of this process is 24359@ip-10-173-29-161 2013-08-10 15:19:38,871 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): hconnection Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2013-08-10 15:19:38,871 DEBUG [main] zookeeper.ZKUtil(229): hconnection Set watcher on existing znode /hbase/master 2013-08-10 15:19:38,872 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(343): hconnection-0x14068cf95210007 connected 2013-08-10 15:19:38,873 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 65 byte(s) of data from znode /hbase/master and set watcher; \x00\x00ip-10-173-29-161.us-w... 2013-08-10 15:19:38,874 DEBUG [main] zookeeper.ZKUtil(229): hconnection-0x14068cf95210007 Set watcher on existing znode /hbase/root-region-server 2013-08-10 15:19:38,875 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:38,877 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=e99afb87-3e83-4509-8792-23cd4... 2013-08-10 15:19:38,897 DEBUG [IPC Server handler 4 on 45825] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210004 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:38,897 DEBUG [IPC Server handler 4 on 45825] client.HConnectionManager$HConnectionImplementation(813): Lookedup root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@69ce835b; serverName=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:38,899 DEBUG [IPC Server handler 4 on 45825] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210004 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:38,899 DEBUG [IPC Server handler 4 on 45825] client.HConnectionManager$HConnectionImplementation(813): Lookedup root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@69ce835b; serverName=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:38,903 DEBUG [IPC Server handler 4 on 45825] catalog.CatalogTracker(542): Set new cached META location: ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:38,903 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:38,903 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:38,907 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:38,914 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.CreateTableHandler(123): Attemping to create the table observed_table 2013-08-10 15:19:38,916 DEBUG [main] client.MetaScanner(203): Scanning .META. starting at row=observed_table,,00000000000000 for max=2147483647 rows using org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 2013-08-10 15:19:38,918 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:38,918 DEBUG [main] client.HConnectionManager$HConnectionImplementation(813): Lookedup root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0; serverName=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:38,925 DEBUG [main] client.HConnectionManager$HConnectionImplementation(1216): Cached location for .META.,,1.1028785192 is ip-10-173-29-161.us-west-1.compute.internal:34018 2013-08-10 15:19:38,948 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(3216): creating HRegion observed_table HTD == {NAME => 'observed_table', FAMILIES => [{NAME => 'fam1', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]} RootDir = hdfs://localhost:34892/user/ec2-user/hbase Table name == observed_table 2013-08-10 15:19:38,950 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(435): FileSystem doesn't support getDefaultBlockSize 2013-08-10 15:19:38,958 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(406): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms 2013-08-10 15:19:38,961 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.SequenceFileLogWriter(126): using new createWriter -- HADOOP-6840 2013-08-10 15:19:38,961 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.SequenceFileLogWriter(136): Path=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/ad8423ae7c8f0e2c584d8e11d28ea996/.logs/hlog.1376147978958, syncFs=true, hflush=false 2013-08-10 15:19:38,961 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(620): for /user/ec2-user/hbase/observed_table/ad8423ae7c8f0e2c584d8e11d28ea996/.logs/hlog.1376147978958 2013-08-10 15:19:38,962 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(478): Using getNumCurrentReplicas--HDFS-826 2013-08-10 15:19:38,962 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:38,962 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(380): Instantiated observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:38,989 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(492): Onlined observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.; next sequenceid=1 2013-08-10 15:19:38,989 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(761): Closing observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.: disabling compactions & flushes 2013-08-10 15:19:38,990 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:38,990 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.Store(482): closed fam1 2013-08-10 15:19:38,990 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(808): Closed observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:38,990 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer] wal.HLog$LogSyncer(1105): MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer interrupted while waiting for sync requests 2013-08-10 15:19:38,991 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer] wal.HLog$LogSyncer(1107): MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer exiting 2013-08-10 15:19:38,991 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(938): closing hlog writer in hdfs://localhost:34892/user/ec2-user/hbase/observed_table/ad8423ae7c8f0e2c584d8e11d28ea996/.logs 2013-08-10 15:19:39,005 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(905): Moved 1 log files to /user/ec2-user/hbase/observed_table/ad8423ae7c8f0e2c584d8e11d28ea996/.oldlogs 2013-08-10 15:19:39,013 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] catalog.MetaEditor(163): Added 1 regions in META 2013-08-10 15:19:39,014 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager(2172): Bulk assigning 1 region(s) round-robin across 2 server(s) 2013-08-10 15:19:39,017 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager$StartupBulkAssigner(2293): Timeout-on-RIT=0 2013-08-10 15:19:39,017 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-1] master.AssignmentManager(1376): Bulk assigning 1 region(s) to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,017 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager(2177): Bulk assigning done 2013-08-10 15:19:39,019 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-1] zookeeper.ZKAssign(169): master:45825-0x14068cf95210000 Async create of unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 with OFFLINE state 2013-08-10 15:19:39,022 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:39,023 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,025 DEBUG [main-EventThread] master.AssignmentManager$CreateUnassignedAsyncCallback(1482): rs=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. state=OFFLINE, ts=1376147979019, server=null, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,027 DEBUG [main-EventThread] master.AssignmentManager$ExistsUnassignedAsyncCallback(1513): rs=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. state=OFFLINE, ts=1376147979019, server=null 2013-08-10 15:19:39,027 INFO [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-1] master.AssignmentManager(1409): ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 unassigned znodes=1 of total=1 2013-08-10 15:19:39,027 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-1] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,029 INFO [PRI IPC Server handler 9 on 37223] regionserver.HRegionServer(2512): Received request to open 1 region(s) 2013-08-10 15:19:39,029 INFO [PRI IPC Server handler 9 on 37223] regionserver.HRegionServer(2475): Received request to open region: observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,037 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:39,037 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-1] master.AssignmentManager(1453): Bulk assigning done for ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,039 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_OFFLINE 2013-08-10 15:19:39,044 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,044 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:39,044 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(3276): Opening region: {NAME => 'observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.', STARTKEY => '', ENDKEY => '', ENCODED => ad8423ae7c8f0e2c584d8e11d28ea996,} 2013-08-10 15:19:39,045 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:39,045 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:39,045 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,046 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(380): Instantiated observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,053 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(492): Onlined observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.; next sequenceid=1 2013-08-10 15:19:39,054 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:39,055 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:39,057 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,057 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:39,058 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] regionserver.HRegionServer(1478): Post open deploy tasks for region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., daughter=false 2013-08-10 15:19:39,058 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:39,059 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,060 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210006 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:39,061 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] client.HConnectionManager$HConnectionImplementation(813): Lookedup root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@40f23491; serverName=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,062 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210006 Retrieved 36 byte(s) of data from znode /hbase/hbaseid; data=e99afb87-3e83-4509-8792-23cd4... 2013-08-10 15:19:39,068 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] client.HConnectionManager$HConnectionImplementation(1216): Cached location for .META.,,1.1028785192 is ip-10-173-29-161.us-west-1.compute.internal:34018 2013-08-10 15:19:39,075 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] catalog.MetaEditor(261): Updated row observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. with server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,076 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] regionserver.HRegionServer(1503): Done with post open deploy task for region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., daughter=false 2013-08-10 15:19:39,076 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:39,077 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:39,080 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,080 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:39,080 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] handler.OpenRegionHandler(284): region transitioned to opened in zookeeper: {NAME => 'observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.', STARTKEY => '', ENDKEY => '', ENCODED => ad8423ae7c8f0e2c584d8e11d28ea996,}, server: ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,081 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] handler.OpenRegionHandler(140): Opened observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. on server:ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,081 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:39,082 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENED, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,082 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] handler.OpenedRegionHandler(147): Handling OPENED event for observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. from ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448; deleting unassigned node 2013-08-10 15:19:39,083 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 that is in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:39,084 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:39,086 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,086 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. has been deleted. 2013-08-10 15:19:39,086 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region ad8423ae7c8f0e2c584d8e11d28ea996 in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:39,087 INFO [main-EventThread] master.AssignmentManager(1148): The master has opened the region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. that was online on ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,087 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:39,943 DEBUG [main] client.MetaScanner(203): Scanning .META. starting at row=observed_table,,00000000000000 for max=2147483647 rows using org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 2013-08-10 15:19:39,950 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:39,953 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:39,953 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:39,963 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(97): Attemping to disable table observed_table 2013-08-10 15:19:39,964 INFO [main] client.HBaseAdmin(753): Started disable of observed_table 2013-08-10 15:19:39,967 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 9 byte(s) of data from znode /hbase/table/observed_table; data=DISABLING 2013-08-10 15:19:39,967 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(121): Offlining 1 regions. 2013-08-10 15:19:39,967 DEBUG [main] client.HBaseAdmin(784): Sleeping= 1000ms, waiting for all regions to be disabled in observed_table 2013-08-10 15:19:39,969 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(1966): Starting unassignment of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. (offlining) 2013-08-10 15:19:39,969 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] zookeeper.ZKAssign(561): master:45825-0x14068cf95210000 Creating unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 in a CLOSING state 2013-08-10 15:19:39,971 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:39,972 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,973 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,973 INFO [PRI IPC Server handler 2 on 37223] regionserver.HRegionServer(2543): Received close region: observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.. Version of ZK closing node:0 2013-08-10 15:19:39,975 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] handler.CloseRegionHandler(105): Processing close of observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,975 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2046): Sent CLOSE to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,976 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(761): Closing observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.: disabling compactions & flushes 2013-08-10 15:19:39,976 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,976 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.Store(482): closed fam1 2013-08-10 15:19:39,977 INFO [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] regionserver.HRegion(808): Closed observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,977 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,978 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_CLOSING 2013-08-10 15:19:39,981 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,981 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,981 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] handler.CloseRegionHandler(140): set region closed state in zk successfully for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. sn name: ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:39,981 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-0] handler.CloseRegionHandler(149): Closed region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,982 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,982 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_CLOSED, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,984 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.ClosedRegionHandler(92): Handling CLOSED event for ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,984 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager(1294): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:39,985 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 that is in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,986 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,988 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:39,988 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. has been deleted. 2013-08-10 15:19:39,988 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region ad8423ae7c8f0e2c584d8e11d28ea996 in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:39,989 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:40,024 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(137): Disabled table is done=true 2013-08-10 15:19:40,969 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:40,970 INFO [main] client.HBaseAdmin(800): Disabled observed_table 2013-08-10 15:19:40,971 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:40,974 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:40,974 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:40,982 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(104): Attemping to enable the table observed_table 2013-08-10 15:19:40,982 INFO [main] client.HBaseAdmin(685): Started enable of observed_table 2013-08-10 15:19:40,983 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=ENABLING 2013-08-10 15:19:40,983 DEBUG [main] client.HBaseAdmin(643): Sleeping= 1000ms, waiting for all regions to be enabled in observed_table 2013-08-10 15:19:40,985 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:40,985 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:40,990 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:40,990 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(132): Table has 1 regions of which 1 are offline. 2013-08-10 15:19:40,992 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] zookeeper.ZKAssign(264): master:45825-0x14068cf95210000 Creating (or updating) unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 with OFFLINE state 2013-08-10 15:19:40,995 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:40,997 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.AssignmentManager(1872): No previous transition plan was found (or we are ignoring an existing plan) for observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. so generated a random one; hri=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., src=, dest=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396; 2 (online=2, available=2) available servers 2013-08-10 15:19:40,997 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.AssignmentManager(1619): Assigning region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:40,997 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:40,997 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:40,998 INFO [PRI IPC Server handler 8 on 34018] regionserver.HRegionServer(2475): Received request to open region: observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:41,006 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:41,007 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_OFFLINE 2013-08-10 15:19:41,009 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,010 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:41,010 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(3276): Opening region: {NAME => 'observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.', STARTKEY => '', ENDKEY => '', ENCODED => ad8423ae7c8f0e2c584d8e11d28ea996,} 2013-08-10 15:19:41,011 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:41,011 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:41,011 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,011 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(380): Instantiated observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:41,019 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(492): Onlined observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.; next sequenceid=1 2013-08-10 15:19:41,019 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:41,020 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:41,023 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,023 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:41,024 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] regionserver.HRegionServer(1478): Post open deploy tasks for region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., daughter=false 2013-08-10 15:19:41,024 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:41,025 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,026 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210005 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:41,026 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] client.HConnectionManager$HConnectionImplementation(813): Lookedup root region location, connection=org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@26fae93e; serverName=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:41,029 DEBUG [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] client.HConnectionManager$HConnectionImplementation(1216): Cached location for .META.,,1.1028785192 is ip-10-173-29-161.us-west-1.compute.internal:34018 2013-08-10 15:19:41,036 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] catalog.MetaEditor(261): Updated row observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. with server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:41,036 INFO [PostOpenDeployTasks:ad8423ae7c8f0e2c584d8e11d28ea996] regionserver.HRegionServer(1503): Done with post open deploy task for region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., daughter=false 2013-08-10 15:19:41,037 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:41,038 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:41,042 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,042 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:41,042 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] handler.OpenRegionHandler(284): region transitioned to opened in zookeeper: {NAME => 'observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.', STARTKEY => '', ENDKEY => '', ENCODED => ad8423ae7c8f0e2c584d8e11d28ea996,}, server: ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:41,043 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] handler.OpenRegionHandler(140): Opened observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. on server:ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:41,043 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:41,043 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENED, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,044 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] handler.OpenedRegionHandler(147): Handling OPENED event for observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. from ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396; deleting unassigned node 2013-08-10 15:19:41,044 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 that is in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:41,046 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:41,048 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:41,048 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. has been deleted. 2013-08-10 15:19:41,048 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region ad8423ae7c8f0e2c584d8e11d28ea996 in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:41,048 INFO [main-EventThread] master.AssignmentManager(1148): The master has opened the region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. that was online on ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:41,048 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:41,095 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(148): Enabled table is done=true 2013-08-10 15:19:41,985 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:41,986 INFO [main] client.HBaseAdmin(659): Enabled table observed_table 2013-08-10 15:19:41,987 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:41,990 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:41,990 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:42,014 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(97): Attemping to disable table observed_table 2013-08-10 15:19:42,014 INFO [main] client.HBaseAdmin(753): Started disable of observed_table 2013-08-10 15:19:42,016 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 9 byte(s) of data from znode /hbase/table/observed_table; data=DISABLING 2013-08-10 15:19:42,016 DEBUG [main] client.HBaseAdmin(784): Sleeping= 1000ms, waiting for all regions to be disabled in observed_table 2013-08-10 15:19:42,020 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(121): Offlining 1 regions. 2013-08-10 15:19:42,020 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(1966): Starting unassignment of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. (offlining) 2013-08-10 15:19:42,021 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] zookeeper.ZKAssign(561): master:45825-0x14068cf95210000 Creating unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 in a CLOSING state 2013-08-10 15:19:42,033 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:42,034 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:42,035 INFO [PRI IPC Server handler 5 on 34018] regionserver.HRegionServer(2543): Received close region: observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.. Version of ZK closing node:0 2013-08-10 15:19:42,035 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:42,036 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] handler.CloseRegionHandler(105): Processing close of observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,036 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2046): Sent CLOSE to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,067 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(761): Closing observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996.: disabling compactions & flushes 2013-08-10 15:19:42,067 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,067 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.Store(482): closed fam1 2013-08-10 15:19:42,068 INFO [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] regionserver.HRegion(808): Closed observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,068 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,069 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_CLOSING 2013-08-10 15:19:42,122 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:42,122 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node ad8423ae7c8f0e2c584d8e11d28ea996 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,122 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] handler.CloseRegionHandler(140): set region closed state in zk successfully for region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. sn name: ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:42,122 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-0] handler.CloseRegionHandler(149): Closed region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,123 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 and set watcher; region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,123 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_CLOSED, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:42,124 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-1] handler.ClosedRegionHandler(92): Handling CLOSED event for ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:42,125 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-1] master.AssignmentManager(1294): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. 2013-08-10 15:19:42,125 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-1] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for ad8423ae7c8f0e2c584d8e11d28ea996 that is in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,126 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-1] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996; data=region=observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,145 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/ad8423ae7c8f0e2c584d8e11d28ea996 2013-08-10 15:19:42,145 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-1] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region ad8423ae7c8f0e2c584d8e11d28ea996 in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:42,145 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. has been deleted. 2013-08-10 15:19:42,146 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:42,174 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(137): Disabled table is done=true 2013-08-10 15:19:43,017 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:43,018 INFO [main] client.HBaseAdmin(800): Disabled observed_table 2013-08-10 15:19:43,019 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:43,021 DEBUG [IPC Server handler 0 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:43,022 DEBUG [IPC Server handler 0 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:43,032 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.TableEventHandler(91): Handling table operation C_M_MODIFY_TABLE on table observed_table 2013-08-10 15:19:43,033 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:43,033 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:43,038 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:43,039 DEBUG [IPC Server handler 2 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:43,039 DEBUG [IPC Server handler 2 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:43,061 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000002 2013-08-10 15:19:43,063 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.TableEventHandler(91): Handling table operation C_M_DELETE_TABLE on table observed_table 2013-08-10 15:19:43,063 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:43,064 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:43,068 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:43,069 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DeleteTableHandler(66): Deleting region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. from META and FS 2013-08-10 15:19:43,072 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] catalog.MetaEditor(276): Deleted region observed_table,,1376147978889.ad8423ae7c8f0e2c584d8e11d28ea996. from META 2013-08-10 15:19:44,068 INFO [main] client.HBaseAdmin(567): Deleted observed_table 2013-08-10 15:19:44,070 DEBUG [main] zookeeper.ZKUtil(96): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 opening connection to ZooKeeper with ensemble (localhost:21818) 2013-08-10 15:19:44,071 INFO [main] zookeeper.RecoverableZooKeeper(90): The identifier of this process is 24359@ip-10-173-29-161 2013-08-10 15:19:44,071 DEBUG [main] catalog.CatalogTracker(232): Starting catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@7952506e 2013-08-10 15:19:44,074 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2013-08-10 15:19:44,074 DEBUG [main] zookeeper.ZKUtil(229): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 Set watcher on existing znode /hbase/root-region-server 2013-08-10 15:19:44,075 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(343): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0-0x14068cf95210008 connected 2013-08-10 15:19:44,075 DEBUG [main] zookeeper.ZKUtil(1122): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0-0x14068cf95210008 Retrieved 63 byte(s) of data from znode /hbase/root-region-server and set watcher; ip-10-173-29-161.us-west-1.co... 2013-08-10 15:19:44,076 DEBUG [main] zookeeper.ZKUtil(231): catalogtracker-on-org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0-0x14068cf95210008 /hbase/unassigned/1028785192 does not exist. Watcher is set. 2013-08-10 15:19:44,077 DEBUG [main] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:44,077 DEBUG [main] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:44,081 DEBUG [main] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:44,081 DEBUG [main] catalog.CatalogTracker(249): Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@7952506e 2013-08-10 15:19:44,086 DEBUG [IPC Server handler 3 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:44,086 DEBUG [IPC Server handler 3 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:44,090 DEBUG [IPC Server handler 3 on 45825] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:44,095 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.CreateTableHandler(123): Attemping to create the table observed_table 2013-08-10 15:19:44,095 DEBUG [main] client.MetaScanner(203): Scanning .META. starting at row=observed_table,,00000000000000 for max=2147483647 rows using org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 2013-08-10 15:19:44,514 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(3216): creating HRegion observed_table HTD == {NAME => 'observed_table', MAX_FILESIZE => '536870912', FAMILIES => [{NAME => 'fam1', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}]} RootDir = hdfs://localhost:34892/user/ec2-user/hbase Table name == observed_table 2013-08-10 15:19:44,516 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(435): FileSystem doesn't support getDefaultBlockSize 2013-08-10 15:19:44,522 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(406): HLog configuration: blocksize=64 MB, rollsize=60.8 MB, enabled=true, optionallogflushinternal=1000ms 2013-08-10 15:19:44,525 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.SequenceFileLogWriter(126): using new createWriter -- HADOOP-6840 2013-08-10 15:19:44,526 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.SequenceFileLogWriter(136): Path=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/07142919e94573a885a7209f5b122079/.logs/hlog.1376147984523, syncFs=true, hflush=false 2013-08-10 15:19:44,526 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(620): for /user/ec2-user/hbase/observed_table/07142919e94573a885a7209f5b122079/.logs/hlog.1376147984523 2013-08-10 15:19:44,526 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(478): Using getNumCurrentReplicas--HDFS-826 2013-08-10 15:19:44,526 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:44,527 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(380): Instantiated observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:44,554 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(492): Onlined observed_table,,1376147984085.07142919e94573a885a7209f5b122079.; next sequenceid=1 2013-08-10 15:19:44,555 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(761): Closing observed_table,,1376147984085.07142919e94573a885a7209f5b122079.: disabling compactions & flushes 2013-08-10 15:19:44,555 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:44,555 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.Store(482): closed fam1 2013-08-10 15:19:44,555 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] regionserver.HRegion(808): Closed observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:44,555 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer] wal.HLog$LogSyncer(1105): MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer interrupted while waiting for sync requests 2013-08-10 15:19:44,556 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer] wal.HLog$LogSyncer(1107): MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0.logSyncer exiting 2013-08-10 15:19:44,556 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(938): closing hlog writer in hdfs://localhost:34892/user/ec2-user/hbase/observed_table/07142919e94573a885a7209f5b122079/.logs 2013-08-10 15:19:44,569 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] wal.HLog(905): Moved 1 log files to /user/ec2-user/hbase/observed_table/07142919e94573a885a7209f5b122079/.oldlogs 2013-08-10 15:19:44,573 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] catalog.MetaEditor(163): Added 1 regions in META 2013-08-10 15:19:44,574 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager(2172): Bulk assigning 1 region(s) round-robin across 2 server(s) 2013-08-10 15:19:44,575 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-0] master.AssignmentManager(1376): Bulk assigning 1 region(s) to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,575 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager$StartupBulkAssigner(2293): Timeout-on-RIT=1000 2013-08-10 15:19:44,575 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-0] zookeeper.ZKAssign(169): master:45825-0x14068cf95210000 Async create of unassigned node for 07142919e94573a885a7209f5b122079 with OFFLINE state 2013-08-10 15:19:44,578 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:44,579 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,580 DEBUG [main-EventThread] master.AssignmentManager$CreateUnassignedAsyncCallback(1482): rs=observed_table,,1376147984085.07142919e94573a885a7209f5b122079. state=OFFLINE, ts=1376147984575, server=null, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,580 DEBUG [main-EventThread] master.AssignmentManager$ExistsUnassignedAsyncCallback(1513): rs=observed_table,,1376147984085.07142919e94573a885a7209f5b122079. state=OFFLINE, ts=1376147984575, server=null 2013-08-10 15:19:44,581 INFO [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-0] master.AssignmentManager(1409): ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 unassigned znodes=1 of total=1 2013-08-10 15:19:44,581 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,582 INFO [PRI IPC Server handler 8 on 34018] regionserver.HRegionServer(2512): Received request to open 1 region(s) 2013-08-10 15:19:44,582 INFO [PRI IPC Server handler 8 on 34018] regionserver.HRegionServer(2475): Received request to open region: observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:44,589 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-StartupBulkAssigner-0] master.AssignmentManager(1453): Bulk assigning done for ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,589 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:44,590 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_OFFLINE 2013-08-10 15:19:44,593 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,593 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:44,593 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(3276): Opening region: {NAME => 'observed_table,,1376147984085.07142919e94573a885a7209f5b122079.', STARTKEY => '', ENDKEY => '', ENCODED => 07142919e94573a885a7209f5b122079,} 2013-08-10 15:19:44,594 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:44,594 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:44,594 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,594 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(380): Instantiated observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:44,602 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(492): Onlined observed_table,,1376147984085.07142919e94573a885a7209f5b122079.; next sequenceid=1 2013-08-10 15:19:44,603 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:44,604 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:44,606 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,606 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:44,606 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] regionserver.HRegionServer(1478): Post open deploy tasks for region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., daughter=false 2013-08-10 15:19:44,607 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:44,607 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,611 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] catalog.MetaEditor(261): Updated row observed_table,,1376147984085.07142919e94573a885a7209f5b122079. with server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,611 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] regionserver.HRegionServer(1503): Done with post open deploy task for region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., daughter=false 2013-08-10 15:19:44,611 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:44,613 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:44,615 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,615 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:44,616 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] handler.OpenRegionHandler(284): region transitioned to opened in zookeeper: {NAME => 'observed_table,,1376147984085.07142919e94573a885a7209f5b122079.', STARTKEY => '', ENDKEY => '', ENCODED => 07142919e94573a885a7209f5b122079,}, server: ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,616 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] handler.OpenRegionHandler(140): Opened observed_table,,1376147984085.07142919e94573a885a7209f5b122079. on server:ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,616 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:44,616 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENED, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,617 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-4] handler.OpenedRegionHandler(147): Handling OPENED event for observed_table,,1376147984085.07142919e94573a885a7209f5b122079. from ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396; deleting unassigned node 2013-08-10 15:19:44,617 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-4] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for 07142919e94573a885a7209f5b122079 that is in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:44,619 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-4] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:44,629 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:44,629 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. has been deleted. 2013-08-10 15:19:44,629 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-4] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region 07142919e94573a885a7209f5b122079 in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:44,629 INFO [main-EventThread] master.AssignmentManager(1148): The master has opened the region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. that was online on ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:44,629 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] master.AssignmentManager(2177): Bulk assigning done 2013-08-10 15:19:44,630 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:45,100 DEBUG [main] client.MetaScanner(203): Scanning .META. starting at row=observed_table,,00000000000000 for max=2147483647 rows using org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation@476ff9e0 2013-08-10 15:19:45,106 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:45,108 DEBUG [IPC Server handler 3 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:45,109 DEBUG [IPC Server handler 3 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:45,118 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(97): Attemping to disable table observed_table 2013-08-10 15:19:45,118 INFO [main] client.HBaseAdmin(753): Started disable of observed_table 2013-08-10 15:19:45,119 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 9 byte(s) of data from znode /hbase/table/observed_table; data=DISABLING 2013-08-10 15:19:45,119 DEBUG [main] client.HBaseAdmin(784): Sleeping= 1000ms, waiting for all regions to be disabled in observed_table 2013-08-10 15:19:45,121 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(121): Offlining 1 regions. 2013-08-10 15:19:45,121 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(1966): Starting unassignment of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. (offlining) 2013-08-10 15:19:45,122 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] zookeeper.ZKAssign(561): master:45825-0x14068cf95210000 Creating unassigned node for 07142919e94573a885a7209f5b122079 in a CLOSING state 2013-08-10 15:19:45,124 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:45,124 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:45,125 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:45,125 INFO [PRI IPC Server handler 7 on 34018] regionserver.HRegionServer(2543): Received close region: observed_table,,1376147984085.07142919e94573a885a7209f5b122079.. Version of ZK closing node:0 2013-08-10 15:19:45,127 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] handler.CloseRegionHandler(105): Processing close of observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,128 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2046): Sent CLOSE to ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,128 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(761): Closing observed_table,,1376147984085.07142919e94573a885a7209f5b122079.: disabling compactions & flushes 2013-08-10 15:19:45,128 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,129 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.Store(482): closed fam1 2013-08-10 15:19:45,129 INFO [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] regionserver.HRegion(808): Closed observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,129 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(757): regionserver:34018-0x14068cf95210001 Attempting to transition node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,130 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKUtil(1122): regionserver:34018-0x14068cf95210001 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_CLOSING 2013-08-10 15:19:45,134 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:45,134 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] zookeeper.ZKAssign(820): regionserver:34018-0x14068cf95210001 Successfully transitioned node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,135 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] handler.CloseRegionHandler(140): set region closed state in zk successfully for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. sn name: ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396 2013-08-10 15:19:45,135 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396-1] handler.CloseRegionHandler(149): Closed region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,135 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,136 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_CLOSED, server=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:45,136 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] handler.ClosedRegionHandler(92): Handling CLOSED event for 07142919e94573a885a7209f5b122079 2013-08-10 15:19:45,136 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] master.AssignmentManager(1294): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:45,137 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for 07142919e94573a885a7209f5b122079 that is in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,138 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,34018,1376147970396, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,139 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:45,140 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. has been deleted. 2013-08-10 15:19:45,140 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-2] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region 07142919e94573a885a7209f5b122079 in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:45,140 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:45,175 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(137): Disabled table is done=true 2013-08-10 15:19:46,121 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:46,121 INFO [main] client.HBaseAdmin(800): Disabled observed_table 2013-08-10 15:19:46,122 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:46,124 DEBUG [IPC Server handler 2 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,125 DEBUG [IPC Server handler 2 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,135 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.TableEventHandler(91): Handling table operation C_M_MODIFY_TABLE on table observed_table 2013-08-10 15:19:46,136 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,136 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,138 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,138 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,143 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:46,145 INFO [IPC Server handler 1 on 45825] handler.TableEventHandler(91): Handling table operation C_M_ADD_FAMILY on table observed_table 2013-08-10 15:19:46,146 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,146 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,151 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:46,152 INFO [IPC Server handler 1 on 45825] master.MasterFileSystem(523): AddColumn. Table = observed_table HCD = {NAME => 'fam2', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '3', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} 2013-08-10 15:19:46,157 DEBUG [IPC Server handler 1 on 45825] util.FSTableDescriptors(489): hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tmp/.tableinfo.0000000002 exists; retrying up to 10 times 2013-08-10 15:19:46,177 INFO [IPC Server handler 1 on 45825] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000003 2013-08-10 15:19:46,197 INFO [IPC Server handler 1 on 45825] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000004 2013-08-10 15:19:46,201 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,201 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,206 INFO [IPC Server handler 4 on 45825] handler.TableEventHandler(91): Handling table operation C_M_MODIFY_FAMILY on table observed_table 2013-08-10 15:19:46,207 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,207 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,212 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:46,212 INFO [IPC Server handler 4 on 45825] master.MasterFileSystem(500): AddModifyColumn. Table = observed_table HCD = {NAME => 'fam2', BLOOMFILTER => 'NONE', REPLICATION_SCOPE => '0', VERSIONS => '25', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => '2147483647', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'} 2013-08-10 15:19:46,241 INFO [IPC Server handler 4 on 45825] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000005 2013-08-10 15:19:46,261 INFO [IPC Server handler 4 on 45825] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000006 2013-08-10 15:19:46,266 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,266 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,308 INFO [main] client.HBaseAdmin(685): Started enable of observed_table 2013-08-10 15:19:46,309 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=ENABLING 2013-08-10 15:19:46,310 DEBUG [main] client.HBaseAdmin(643): Sleeping= 1000ms, waiting for all regions to be enabled in observed_table 2013-08-10 15:19:46,564 WARN [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] util.FSTableDescriptors(510): Failed delete of hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000001; continuing 2013-08-10 15:19:46,564 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] util.FSTableDescriptors(440): Updated tableinfo=hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000002 2013-08-10 15:19:46,567 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] util.FSTableDescriptors(287): Cleaned up old tableinfo file hdfs://localhost:34892/user/ec2-user/hbase/observed_table/.tableinfo.0000000002 2013-08-10 15:19:46,568 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(104): Attemping to enable the table observed_table 2013-08-10 15:19:46,571 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:46,571 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:46,576 DEBUG [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] client.HTable$ClientScanner(1200): Finished with scanning at {NAME => '.META.,,1', STARTKEY => '', ENDKEY => '', ENCODED => 1028785192,} 2013-08-10 15:19:46,576 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(132): Table has 1 regions of which 1 are offline. 2013-08-10 15:19:46,577 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] zookeeper.ZKAssign(264): master:45825-0x14068cf95210000 Creating (or updating) unassigned node for 07142919e94573a885a7209f5b122079 with OFFLINE state 2013-08-10 15:19:46,580 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:46,581 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.AssignmentManager(1872): No previous transition plan was found (or we are ignoring an existing plan) for observed_table,,1376147984085.07142919e94573a885a7209f5b122079. so generated a random one; hri=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., src=, dest=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448; 2 (online=2, available=2) available servers 2013-08-10 15:19:46,581 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.AssignmentManager(1619): Assigning region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,581 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.EnableTableHandler$BulkEnabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,581 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,582 INFO [PRI IPC Server handler 3 on 37223] regionserver.HRegionServer(2475): Received request to open region: observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:46,589 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:46,591 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_OFFLINE 2013-08-10 15:19:46,594 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,594 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_OFFLINE to RS_ZK_REGION_OPENING 2013-08-10 15:19:46,595 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(3276): Opening region: {NAME => 'observed_table,,1376147984085.07142919e94573a885a7209f5b122079.', STARTKEY => '', ENDKEY => '', ENCODED => 07142919e94573a885a7209f5b122079,} 2013-08-10 15:19:46,595 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:46,595 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(386): Setting up tabledescriptor config now ... 2013-08-10 15:19:46,596 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,596 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(380): Instantiated observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:46,608 INFO [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(492): Onlined observed_table,,1376147984085.07142919e94573a885a7209f5b122079.; next sequenceid=1 2013-08-10 15:19:46,608 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:46,609 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:46,613 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,613 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENING 2013-08-10 15:19:46,614 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] regionserver.HRegionServer(1478): Post open deploy tasks for region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., daughter=false 2013-08-10 15:19:46,614 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:46,615 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENING, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,619 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] catalog.MetaEditor(261): Updated row observed_table,,1376147984085.07142919e94573a885a7209f5b122079. with server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,619 INFO [PostOpenDeployTasks:07142919e94573a885a7209f5b122079] regionserver.HRegionServer(1503): Done with post open deploy task for region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., daughter=false 2013-08-10 15:19:46,619 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:46,620 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENING 2013-08-10 15:19:46,622 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,622 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node 07142919e94573a885a7209f5b122079 from RS_ZK_REGION_OPENING to RS_ZK_REGION_OPENED 2013-08-10 15:19:46,623 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] handler.OpenRegionHandler(284): region transitioned to opened in zookeeper: {NAME => 'observed_table,,1376147984085.07142919e94573a885a7209f5b122079.', STARTKEY => '', ENDKEY => '', ENCODED => 07142919e94573a885a7209f5b122079,}, server: ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,623 DEBUG [RS_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] handler.OpenRegionHandler(140): Opened observed_table,,1376147984085.07142919e94573a885a7209f5b122079. on server:ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,623 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:46,624 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_OPENED, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,624 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.OpenedRegionHandler(147): Handling OPENED event for observed_table,,1376147984085.07142919e94573a885a7209f5b122079. from ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448; deleting unassigned node 2013-08-10 15:19:46,624 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for 07142919e94573a885a7209f5b122079 that is in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:46,625 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_OPENED 2013-08-10 15:19:46,627 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:46,627 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. has been deleted. 2013-08-10 15:19:46,627 DEBUG [MASTER_OPEN_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region 07142919e94573a885a7209f5b122079 in expected state RS_ZK_REGION_OPENED 2013-08-10 15:19:46,627 INFO [main-EventThread] master.AssignmentManager(1148): The master has opened the region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. that was online on ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:46,628 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:46,686 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.EnableTableHandler(148): Enabled table is done=true 2013-08-10 15:19:47,311 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:47,311 INFO [main] client.HBaseAdmin(659): Enabled table observed_table 2013-08-10 15:19:47,312 DEBUG [main] zookeeper.ZKUtil(467): hconnection-0x14068cf95210007 Unable to get data of znode /hbase/table/observed_table because node does not exist (not an error) 2013-08-10 15:19:47,315 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:47,315 DEBUG [IPC Server handler 1 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,' 2013-08-10 15:19:47,369 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(97): Attemping to disable table observed_table 2013-08-10 15:19:47,369 INFO [main] client.HBaseAdmin(753): Started disable of observed_table 2013-08-10 15:19:47,370 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 9 byte(s) of data from znode /hbase/table/observed_table; data=DISABLING 2013-08-10 15:19:47,371 DEBUG [main] client.HBaseAdmin(784): Sleeping= 1000ms, waiting for all regions to be disabled in observed_table 2013-08-10 15:19:47,372 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(121): Offlining 1 regions. 2013-08-10 15:19:47,373 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(1966): Starting unassignment of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. (offlining) 2013-08-10 15:19:47,373 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] zookeeper.ZKAssign(561): master:45825-0x14068cf95210000 Creating unassigned node for 07142919e94573a885a7209f5b122079 in a CLOSING state 2013-08-10 15:19:47,375 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:47,376 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.ServerManager(533): New connection to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:47,377 DEBUG [main-EventThread] zookeeper.ZKUtil(229): master:45825-0x14068cf95210000 Set watcher on existing znode /hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:47,377 INFO [PRI IPC Server handler 4 on 37223] regionserver.HRegionServer(2543): Received close region: observed_table,,1376147984085.07142919e94573a885a7209f5b122079.. Version of ZK closing node:0 2013-08-10 15:19:47,378 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] handler.CloseRegionHandler(105): Processing close of observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,378 DEBUG [ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-org.apache.hadoop.hbase.master.handler.DisableTableHandler$BulkDisabler-0] master.AssignmentManager(2046): Sent CLOSE to ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,379 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(761): Closing observed_table,,1376147984085.07142919e94573a885a7209f5b122079.: disabling compactions & flushes 2013-08-10 15:19:47,379 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(791): Updates disabled for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,379 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.Store(482): closed fam1 2013-08-10 15:19:47,379 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.Store(482): closed fam2 2013-08-10 15:19:47,380 INFO [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] regionserver.HRegion(808): Closed observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,380 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(757): regionserver:37223-0x14068cf95210002 Attempting to transition node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,381 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKUtil(1122): regionserver:37223-0x14068cf95210002 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158, state=M_ZK_REGION_CLOSING 2013-08-10 15:19:47,383 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:47,383 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] zookeeper.ZKAssign(820): regionserver:37223-0x14068cf95210002 Successfully transitioned node 07142919e94573a885a7209f5b122079 from M_ZK_REGION_CLOSING to RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,384 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] handler.CloseRegionHandler(140): set region closed state in zk successfully for region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. sn name: ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448 2013-08-10 15:19:47,384 DEBUG [RS_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448-1] handler.CloseRegionHandler(149): Closed region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,384 DEBUG [main-EventThread] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079 and set watcher; region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,385 DEBUG [main-EventThread] master.AssignmentManager(709): Handling transition=RS_ZK_REGION_CLOSED, server=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, region=07142919e94573a885a7209f5b122079 2013-08-10 15:19:47,385 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] handler.ClosedRegionHandler(92): Handling CLOSED event for 07142919e94573a885a7209f5b122079 2013-08-10 15:19:47,385 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] master.AssignmentManager(1294): Table being disabled so deleting ZK node and removing from regions in transition, skipping assignment of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. 2013-08-10 15:19:47,385 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKAssign(483): master:45825-0x14068cf95210000 Deleting existing unassigned node for 07142919e94573a885a7209f5b122079 that is in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,386 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKUtil(1122): master:45825-0x14068cf95210000 Retrieved 142 byte(s) of data from znode /hbase/unassigned/07142919e94573a885a7209f5b122079; data=region=observed_table,,1376147984085.07142919e94573a885a7209f5b122079., origin=ip-10-173-29-161.us-west-1.compute.internal,37223,1376147970448, state=RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,389 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/unassigned/07142919e94573a885a7209f5b122079 2013-08-10 15:19:47,389 DEBUG [main-EventThread] master.AssignmentManager(1136): The znode of region observed_table,,1376147984085.07142919e94573a885a7209f5b122079. has been deleted. 2013-08-10 15:19:47,389 DEBUG [MASTER_CLOSE_REGION-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-3] zookeeper.ZKAssign(512): master:45825-0x14068cf95210000 Successfully deleted unassigned node for region 07142919e94573a885a7209f5b122079 in expected state RS_ZK_REGION_CLOSED 2013-08-10 15:19:47,389 DEBUG [main-EventThread] zookeeper.ZooKeeperWatcher(266): master:45825-0x14068cf95210000 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/unassigned 2013-08-10 15:19:47,431 INFO [MASTER_TABLE_OPERATIONS-ip-10-173-29-161.us-west-1.compute.internal,45825,1376147970158-0] handler.DisableTableHandler(137): Disabled table is done=true 2013-08-10 15:19:48,372 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:48,373 INFO [main] client.HBaseAdmin(800): Disabled observed_table 2013-08-10 15:19:48,374 DEBUG [main] zookeeper.ZKUtil(1122): hconnection-0x14068cf95210007 Retrieved 8 byte(s) of data from znode /hbase/table/observed_table; data=DISABLED 2013-08-10 15:19:48,376 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1120): Creating scanner over .META. starting at key 'observed_table,,' 2013-08-10 15:19:48,376 DEBUG [IPC Server handler 4 on 45825] client.HTable$ClientScanner(1213): Advancing internal scanner to startKey at 'observed_table,,'