2020-11-06 02:01:26,420 WARN [PEWorker-12] assignment.RegionTransitionProcedure: Failed transition, suspend 1secs pid=12, ppid=9, state=RUNNABLE:REGION_TRANSITION_QUEUE, locked=true; AssignProcedure table=TestTable, region=37d62d2c1934da269a592e0e5cbca82a; rit=OFFLINE, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.master.TableStateManager$TableStateNotFoundException: TestTable at org.apache.hadoop.hbase.master.TableStateManager.getTableState(TableStateManager.java:215) at org.apache.hadoop.hbase.master.assignment.AssignProcedure.assign(AssignProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.AssignProcedure.startTransition(AssignProcedure.java:205) at org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure.execute(RegionTransitionProcedure.java:355) at org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure.execute(RegionTransitionProcedure.java:97) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:957) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1835) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1595) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1200(ProcedureExecutor.java:80) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2140) 2020-11-06 02:01:26,420 WARN [PEWorker-11] assignment.RegionTransitionProcedure: Failed transition, suspend 1secs pid=11, ppid=9, state=RUNNABLE:REGION_TRANSITION_QUEUE, locked=true; AssignProcedure table=hbase:namespace, region=d1e4c3b1c102eb380c4257afc2493aca; rit=OFFLINE, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.master.TableStateManager$TableStateNotFoundException: hbase:namespace at org.apache.hadoop.hbase.master.TableStateManager.getTableState(TableStateManager.java:215) at org.apache.hadoop.hbase.master.assignment.AssignProcedure.assign(AssignProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.AssignProcedure.startTransition(AssignProcedure.java:205) at org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure.execute(RegionTransitionProcedure.java:355) at org.apache.hadoop.hbase.master.assignment.RegionTransitionProcedure.execute(RegionTransitionProcedure.java:97) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:957) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1835) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1595) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1200(ProcedureExecutor.java:80) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2140) 2020-11-06 02:01:26,422 INFO [PEWorker-12] procedure2.TimeoutExecutorThread: ADDED pid=12, ppid=9, state=WAITING_TIMEOUT:REGION_TRANSITION_QUEUE, locked=true; AssignProcedure table=TestTable, region=37d62d2c1934da269a592e0e5cbca82a; timeout=1002, timestamp=1604628087424 2020-11-06 02:01:26,423 INFO [PEWorker-11] procedure2.TimeoutExecutorThread: ADDED pid=11, ppid=9, state=WAITING_TIMEOUT:REGION_TRANSITION_QUEUE, locked=true; AssignProcedure table=hbase:namespace, region=d1e4c3b1c102eb380c4257afc2493aca; timeout=1001, timestamp=1604628087424 2020-11-06 02:01:26,428 INFO [PEWorker-10] master.MasterWalManager: Log dir for server hbase-master-single-node-n3.hbase_dc1ring,16020,1604627809473 does not exist 2020-11-06 02:01:26,429 INFO [PEWorker-10] master.SplitLogManager: Finished splitting (more than or equal to) 0 bytes in 0 log files in [] in 0ms 2020-11-06 02:01:26,438 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=16, state=SUCCESS; ServerCrashProcedure server=hbase-master-single-node-n3.hbase_dc1ring,16020,1604627809473, splitWal=true, meta=true in 2mins, 39.674sec 2020-11-06 02:01:26,576 INFO [master/399fd6ca0c6d:16000:becomeActiveMaster] hbase.MetaTableAccessor: Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2020-11-06 02:01:26,643 INFO [master/399fd6ca0c6d:16000:becomeActiveMaster] master.TableStateManager: Migrating table state from zookeeper to hbase:meta; tableName=TestTable, state=ENABLED 2020-11-06 02:01:26,652 INFO [master/399fd6ca0c6d:16000:becomeActiveMaster] hbase.MetaTableAccessor: Updated tableName=TestTable, state=ENABLED in hbase:meta 2020-11-06 02:01:26,783 WARN [master/399fd6ca0c6d:16000:becomeActiveMaster] master.TableNamespaceManager: Caught exception in initializing namespace table manager org.apache.hadoop.hbase.TableNotFoundException: hbase:namespace at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:864) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:759) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:745) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:716) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.getRegionLocation(ConnectionImplementation.java:594) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.getRegionLocation(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:72) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:223) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:105) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:386) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:360) at org.apache.hadoop.hbase.master.TableNamespaceManager.get(TableNamespaceManager.java:142) at org.apache.hadoop.hbase.master.TableNamespaceManager.isTableAvailableAndInitialized(TableNamespaceManager.java:279) at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:104) at org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:63) at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:226) at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1251) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1031) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2254) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:583) at java.lang.Thread.run(Thread.java:748) 2020-11-06 02:01:26,790 ERROR [master/399fd6ca0c6d:16000:becomeActiveMaster] master.HMaster: Failed to become active master java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:345) at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:291) at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1253) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1031) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2254) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:583) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.TableNotFoundException: hbase:namespace at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:864) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:759) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:745) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:716) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.getRegionLocation(ConnectionImplementation.java:594) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.getRegionLocation(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:72) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:223) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:105) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:386) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:360) at org.apache.hadoop.hbase.master.TableNamespaceManager.get(TableNamespaceManager.java:142) at org.apache.hadoop.hbase.master.TableNamespaceManager.isTableAvailableAndInitialized(TableNamespaceManager.java:279) at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:104) at org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:63) at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:226) at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1251) ... 4 more 2020-11-06 02:01:26,791 ERROR [master/399fd6ca0c6d:16000:becomeActiveMaster] master.HMaster: Master server abort: loaded coprocessors are: [] 2020-11-06 02:01:26,791 ERROR [master/399fd6ca0c6d:16000:becomeActiveMaster] master.HMaster: ***** ABORTING master 399fd6ca0c6d,16000,1604628075265: Unhandled exception. Starting shutdown. ***** java.lang.IllegalStateException: Expected the service ClusterSchemaServiceImpl [FAILED] to be RUNNING, but the service has FAILED at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.checkCurrentState(AbstractService.java:345) at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.awaitRunning(AbstractService.java:291) at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1253) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1031) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2254) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:583) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.TableNotFoundException: hbase:namespace at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegionInMeta(ConnectionImplementation.java:864) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:759) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:745) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.locateRegion(ConnectionImplementation.java:716) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.locateRegion(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.ConnectionImplementation.getRegionLocation(ConnectionImplementation.java:594) at org.apache.hadoop.hbase.client.ConnectionUtils$ShortCircuitingClusterConnection.getRegionLocation(ConnectionUtils.java:131) at org.apache.hadoop.hbase.client.HRegionLocator.getRegionLocation(HRegionLocator.java:72) at org.apache.hadoop.hbase.client.RegionServerCallable.prepare(RegionServerCallable.java:223) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:105) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:386) at org.apache.hadoop.hbase.client.HTable.get(HTable.java:360) at org.apache.hadoop.hbase.master.TableNamespaceManager.get(TableNamespaceManager.java:142) at org.apache.hadoop.hbase.master.TableNamespaceManager.isTableAvailableAndInitialized(TableNamespaceManager.java:279) at org.apache.hadoop.hbase.master.TableNamespaceManager.start(TableNamespaceManager.java:104) at org.apache.hadoop.hbase.master.ClusterSchemaServiceImpl.doStart(ClusterSchemaServiceImpl.java:63) at org.apache.hbase.thirdparty.com.google.common.util.concurrent.AbstractService.startAsync(AbstractService.java:226) at org.apache.hadoop.hbase.master.HMaster.initClusterSchemaService(HMaster.java:1251) ... 4 more