/hadoop/libexec/hadoop-functions.sh: line 2360: HADOOP_ORG.APACHE.HADOOP.HBASE.UTIL.GETJAVAPROPERTY_USER: invalid variable name /hadoop/libexec/hadoop-functions.sh: line 2455: HADOOP_ORG.APACHE.HADOOP.HBASE.UTIL.GETJAVAPROPERTY_OPTS: invalid variable name 2020-05-21 17:16:35,670 INFO [main] master.HMaster: STARTING service HMaster 2020-05-21 17:16:35,671 INFO [main] util.VersionInfo: HBase 2.2.4 2020-05-21 17:16:35,671 INFO [main] util.VersionInfo: Source code repository file:///hbase-2.2.4 revision=Unknown 2020-05-21 17:16:35,671 INFO [main] util.VersionInfo: Compiled by root on Tue Mar 24 18:56:23 UTC 2020 2020-05-21 17:16:35,671 INFO [main] util.VersionInfo: From source with checksum 811a583f3df897a058639334f00f928d 2020-05-21 17:16:36,257 INFO [main] util.ServerCommandLine: hbase.tmp.dir: /tmp/hbase-hbase 2020-05-21 17:16:36,257 INFO [main] util.ServerCommandLine: hbase.rootdir: hdfs://aeris/hbase 2020-05-21 17:16:36,257 INFO [main] util.ServerCommandLine: hbase.cluster.distributed: true 2020-05-21 17:16:36,257 INFO [main] util.ServerCommandLine: hbase.zookeeper.quorum: zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:PATH=/usr/local/openjdk-8/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/hadoop/bin:/hbase/bin 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:HBASE_LOGFILE=master.log 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_SERVICE_HOST=172.29.167.229 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:JAVA_URL_VERSION=8u242b08 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:KUBERNETES_PORT=tcp://172.29.0.1:443 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:JAVA_HOME=/usr/local/openjdk-8 2020-05-21 17:16:36,258 INFO [main] util.ServerCommandLine: env:KUBERNETES_SERVICE_HOST=172.29.0.1 2020-05-21 17:16:36,261 INFO [main] util.ServerCommandLine: env:LANG=C.UTF-8 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:LD_LIBRARY_PATH=:/hadoop/lib/native:/hadoop/lib/native 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:JVM_PID=7 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBASE_REST_OPTS= 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:JAVA_VERSION=8u242 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:PWD=/ 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_PORT_8080_TCP_PORT=8080 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:KUBERNETES_PORT_443_TCP=tcp://172.29.0.1:443 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBASE_CLASSPATH=/hadoop/etc/hadoop 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_PORT_8080_TCP_PROTO=tcp 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_PORT=tcp://172.29.167.229:8080 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:KUBERNETES_PORT_443_TCP_ADDR=172.29.0.1 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBASE_ENV_INIT=true 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_SERVICE_PORT=8080 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:JAVA_BASE_URL=https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u242-b08/OpenJDK8U-jre_ 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_PORT_8080_TCP_ADDR=172.29.167.229 2020-05-21 17:16:36,262 INFO [main] util.ServerCommandLine: env:KUBERNETES_PORT_443_TCP_PROTO=tcp 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBASE_LOG_DIR=/logs 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: n/../lib/spymemcached-2.12.2.jar:/hbase/bin/../lib/stax2-api-3.1.4.jar:/hbase/bin/../lib/validation-api-1.1.0.Final.jar:/hbase/bin/../lib/woodstox-core-5.0.3.jar:/hbase/bin/../lib/zookeeper-3.4.10.jar:/hbase/bin/../lib/client-facing-thirdparty/audience-annotations-0.5.0.jar:/hbase/bin/../lib/client-facing-thirdparty/commons-logging-1.2.jar:/hbase/bin/../lib/client-facing-thirdparty/findbugs-annotations-1.3.9-1.jar:/hbase/bin/../lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/hbase/bin/../lib/client-facing-thirdparty/log4j-1.2.17.jar:/hbase/bin/../lib/client-facing-thirdparty/slf4j-api-1.7.25.jar:/hadoop/etc/hadoop:/hadoop/share/hadoop/common/lib/*:/hadoop/share/hadoop/common/*:/hadoop/share/hadoop/hdfs:/hadoop/share/hadoop/hdfs/lib/*:/hadoop/share/hadoop/hdfs/*:/hadoop/share/hadoop/mapreduce/lib/*:/hadoop/share/hadoop/mapreduce/*:/hadoop/share/hadoop/yarn/lib/*:/hadoop/share/hadoop/yarn/*:/hadoop/contrib/capacity-scheduler/*.jar:/gcsconnectorjar/*.jar:/hadoop/etc/hadoop 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:SERVER_GC_OPTS=-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/logs/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=4 -XX:GCLogFileSize=256M 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:KUBERNETES_SERVICE_PORT=443 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBASE_LIBRARY_PATH=/hadoop/lib/native 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_PORT_8080_TCP=tcp://172.29.167.229:8080 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HOSTNAME=hbasemaster-0 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HADOOP_HOME=/hadoop 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:KUBERNETES_PORT_443_TCP_PORT=443 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBCK_METRICS_SERVICE_PORT_METRICS=8080 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBASE_HOME=/hbase/bin/.. 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:HBASE_THRIFT_OPTS= 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:KUBERNETES_SERVICE_PORT_HTTPS=443 2020-05-21 17:16:36,263 INFO [main] util.ServerCommandLine: env:SHLVL=0 2020-05-21 17:16:36,264 INFO [main] util.ServerCommandLine: env:HOME=/home/hadoop 2020-05-21 17:16:36,264 INFO [main] util.ServerCommandLine: env:MALLOC_ARENA_MAX=4 2020-05-21 17:16:36,264 INFO [main] util.ServerCommandLine: vmName=OpenJDK 64-Bit Server VM, vmVendor=Oracle Corporation, vmVersion=25.242-b08 2020-05-21 17:16:36,264 INFO [main] util.ServerCommandLine: vmInputArguments=[-Dproc_master, -XX:OnOutOfMemoryError=kill -9 %p, -XX:MaxGCPauseMillis=50, -XX:+UseG1GC, -XX:+ParallelRefProcEnabled, -XX:+PerfDisableSharedMem, -XX:-ResizePLAB, -XX:-OmitStackTraceInFastThrow, -XX:+HeapDumpOnOutOfMemoryError, -Djava.net.preferIPv4Stack=true, -Dsun.net.inetaddr.ttl=10, -verbose:gc, -XX:+PrintGCDetails, -XX:+PrintGCDateStamps, -Xloggc:/logs/gc.log, -XX:+UseGCLogFileRotation, -XX:NumberOfGCLogFiles=4, -XX:GCLogFileSize=256M, -Dproc_hmaster, -Xmx8g, -Dcom.sun.management.jmxremote.ssl=false, -Dcom.sun.management.jmxremote.authenticate=false, -javaagent:/jmxexporterjar/jmx_prometheus_javaagent.jar=7072:/hbase/conf/hbase-jmx-exporter-config.yml, -Dcom.sun.management.jmxremote.port=11101, -Dhbase.log.dir=/logs, -Dhbase.log.file=master.log, -Dhbase.home.dir=/hbase/bin/.., -Dhbase.id.str=, -Dhbase.root.logger=INFO,console, -Djava.library.path=/hadoop/lib/native:/hadoop/lib/native, -Dhbase.security.logger=INFO,RFAS] 2020-05-21 17:16:37,063 INFO [main] metrics.MetricRegistries: Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2020-05-21 17:16:38,097 INFO [main] regionserver.RSRpcServices: master/hbasemaster-0:16000 server-side Connection retries=45 2020-05-21 17:16:38,144 INFO [main] ipc.RpcExecutor: Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=5, maxQueueLength=500, handlerCount=50 2020-05-21 17:16:38,148 INFO [main] ipc.RpcExecutor: Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=500, handlerCount=20 2020-05-21 17:16:38,148 INFO [main] ipc.RWQueueRpcExecutor: priority.RWQ.Fifo writeQueues=1 writeHandlers=2 readQueues=1 readHandlers=18 scanQueues=0 scanHandlers=0 2020-05-21 17:16:38,149 INFO [main] ipc.RpcExecutor: Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=500, handlerCount=3 2020-05-21 17:16:38,149 INFO [main] ipc.RpcExecutor: Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=500, handlerCount=1 2020-05-21 17:16:38,387 INFO [main] ipc.RpcServerFactory: Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientService, hbase.pb.AdminService 2020-05-21 17:16:38,668 INFO [main] ipc.NettyRpcServer: Bind to /10.128.25.30:16000 2020-05-21 17:16:40,163 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2020-05-21 17:16:40,169 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2020-05-21 17:16:40,388 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=master:16000 connecting to ZooKeeper ensemble=zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.10-39d3a4f269333c922ed3db283be479f9deacaa0f, built on 03/23/2017 10:13 GMT 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=hbasemaster-0.hbase.hbase.svc.cluster.local 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.8.0_242 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/local/openjdk-8 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: hare/hadoop/yarn/hadoop-yarn-services-core-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-common-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-common-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-nodemanager-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-applicationhistoryservice-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-timeline-pluginstorage-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-web-proxy-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-applications-unmanaged-am-launcher-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-server-router-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-services-api-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-applications-distributedshell-3.1.3.jar:/hadoop/share/hadoop/yarn/hadoop-yarn-registry-3.1.3.jar:/hadoop/contrib/capacity-scheduler/*.jar:/gcsconnectorjar/*.jar:/hadoop/etc/hadoop:/jmxexporterjar/jmx_prometheus_javaagent.jar 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/hadoop/lib/native:/hadoop/lib/native 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux 2020-05-21 17:16:40,397 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 2020-05-21 17:16:40,398 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=4.14.138+ 2020-05-21 17:16:40,398 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=hbase 2020-05-21 17:16:40,398 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/home/hadoop 2020-05-21 17:16:40,398 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/ 2020-05-21 17:16:40,399 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@162b3d47 2020-05-21 17:16:40,457 INFO [main-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Opening socket connection to server zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181. Will not attempt to authenticate using SASL (unknown error) 2020-05-21 17:16:40,462 INFO [main-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Socket connection established to zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181, initiating session 2020-05-21 17:16:40,485 INFO [main-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Session establishment complete on server zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181, sessionid = 0x10000829d3e2864, negotiated timeout = 40000 2020-05-21 17:16:40,585 INFO [main] util.log: Logging initialized @7086ms 2020-05-21 17:16:40,663 INFO [main] http.HttpRequestLog: Http request log for http.requests.master is not defined 2020-05-21 17:16:40,677 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2020-05-21 17:16:40,677 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2020-05-21 17:16:40,678 INFO [main] http.HttpServer: Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2020-05-21 17:16:40,680 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2020-05-21 17:16:40,680 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2020-05-21 17:16:40,680 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2020-05-21 17:16:40,689 INFO [main] http.HttpServer: ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2020-05-21 17:16:40,759 INFO [main] http.HttpServer: Jetty bound to port 16010 2020-05-21 17:16:40,761 INFO [main] server.Server: jetty-9.3.27.v20190418, build timestamp: 2019-04-18T18:11:38Z, git hash: d3e249f86955d04bc646bb620905b7c1bc596a8d 2020-05-21 17:16:40,800 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2020-05-21 17:16:40,804 INFO [main] handler.ContextHandler: Started o.e.j.s.ServletContextHandler@42c28305{/logs,file:///logs/,AVAILABLE} 2020-05-21 17:16:40,804 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2020-05-21 17:16:40,805 INFO [main] handler.ContextHandler: Started o.e.j.s.ServletContextHandler@7d199c68{/static,file:///hbase/hbase-webapps/static/,AVAILABLE} 2020-05-21 17:16:40,977 INFO [main] webapp.StandardDescriptorProcessor: NO JSP Support for /, did not find org.eclipse.jetty.jsp.JettyJspServlet 2020-05-21 17:16:40,989 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2020-05-21 17:16:41,044 INFO [main] handler.ContextHandler: Started o.e.j.w.WebAppContext@6daf7d37{/,file:///hbase/hbase-webapps/master/,AVAILABLE}{file:/hbase/hbase-webapps/master} 2020-05-21 17:16:41,051 INFO [main] server.AbstractConnector: Started ServerConnector@75961f16{HTTP/1.1,[http/1.1]}{0.0.0.0:16010} 2020-05-21 17:16:41,052 INFO [main] server.Server: Started @7553ms 2020-05-21 17:16:41,056 INFO [main] master.HMaster: hbase.rootdir=hdfs://aeris/hbase, hbase.cluster.distributed=true 2020-05-21 17:16:41,087 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.HMaster: Adding backup master ZNode /hbase/backup-masters/hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313 2020-05-21 17:16:41,159 INFO [master/hbasemaster-0:16000] regionserver.HRegionServer: ClusterId : 248caf68-f7af-424b-9f97-0008069ea72f 2020-05-21 17:16:41,259 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ActiveMasterManager: Another master is the active master, hbasemaster-1.hbase.hbase.svc.cluster.local,16000,1590016843485; waiting to become the next active master 2020-05-21 17:16:46,011 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ActiveMasterManager: Another master is the active master, hbasemaster-1.hbase.hbase.svc.cluster.local,16000,1590081401163; waiting to become the next active master 2020-05-21 17:26:36,195 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313 from backup master directory 2020-05-21 17:26:36,200 WARN [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ZNodeClearer: Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2020-05-21 17:26:36,200 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ActiveMasterManager: Registered as active master=hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313 2020-05-21 17:26:37,218 INFO [master/hbasemaster-0:16000:becomeActiveMaster] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2020-05-21 17:26:37,255 INFO [master/hbasemaster-0:16000:becomeActiveMaster] coordination.SplitLogManagerCoordination: Found 0 orphan tasks and 0 rescan nodes 2020-05-21 17:26:37,256 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: SplitLogManager Timeout Monitor Period: 1000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:37,511 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a] zookeeper.ZooKeeper: Initiating client connection, connectString=zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$72/959144063@7c29f93e 2020-05-21 17:26:37,530 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a-SendThread(zookeeper-1-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Opening socket connection to server zookeeper-1-zookeeper.zookeeper.svc.cluster.local/172.29.120.104:2181. Will not attempt to authenticate using SASL (unknown error) 2020-05-21 17:26:37,532 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a-SendThread(zookeeper-1-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Socket connection established to zookeeper-1-zookeeper.zookeeper.svc.cluster.local/172.29.120.104:2181, initiating session 2020-05-21 17:26:37,535 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a-SendThread(zookeeper-1-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Session establishment complete on server zookeeper-1-zookeeper.zookeeper.svc.cluster.local/172.29.120.104:2181, sessionid = 0x200008282832a6b, negotiated timeout = 40000 2020-05-21 17:26:37,593 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Starting 16 core workers (bigger of cpus/4 or 16) with max (burst) worker count=160 2020-05-21 17:26:37,598 INFO [master/hbasemaster-0:16000:becomeActiveMaster] util.FSHDFSUtils: Recover lease on dfs file hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:26:37,604 INFO [master/hbasemaster-0:16000:becomeActiveMaster] util.FSHDFSUtils: Recovered lease, attempt=0 on file=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log after 6ms 2020-05-21 17:26:37,700 WARN [master/hbasemaster-0:16000:becomeActiveMaster] wal.WALProcedureStore: Unable to read tracker for hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat$InvalidWALDataException: Invalid Trailer version. got 8 expected 1 at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.readTrailer(ProcedureWALFormat.java:186) at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFile.readTrailer(ProcedureWALFile.java:91) at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFile.readTracker(ProcedureWALFile.java:98) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.initOldLog(WALProcedureStore.java:1390) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.initOldLogs(WALProcedureStore.java:1339) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.recoverLease(WALProcedureStore.java:417) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:586) at org.apache.hadoop.hbase.master.HMaster.createProcedureExecutor(HMaster.java:1523) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:938) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2124) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:580) at java.lang.Thread.run(Thread.java:748) 2020-05-21 17:26:37,702 INFO [master/hbasemaster-0:16000:becomeActiveMaster] util.FSHDFSUtils: Recover lease on dfs file hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log 2020-05-21 17:26:37,703 INFO [master/hbasemaster-0:16000:becomeActiveMaster] util.FSHDFSUtils: Recovered lease, attempt=0 on file=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log after 1ms 2020-05-21 17:26:37,791 WARN [master/hbasemaster-0:16000:becomeActiveMaster] wal.WALProcedureStore: Unable to read tracker for hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat$InvalidWALDataException: Invalid Trailer version. got 8 expected 1 at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFormat.readTrailer(ProcedureWALFormat.java:186) at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFile.readTrailer(ProcedureWALFile.java:91) at org.apache.hadoop.hbase.procedure2.store.wal.ProcedureWALFile.readTracker(ProcedureWALFile.java:98) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.initOldLog(WALProcedureStore.java:1390) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.initOldLogs(WALProcedureStore.java:1339) at org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore.recoverLease(WALProcedureStore.java:417) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:586) at org.apache.hadoop.hbase.master.HMaster.createProcedureExecutor(HMaster.java:1523) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:938) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2124) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:580) at java.lang.Thread.run(Thread.java:748) 2020-05-21 17:26:37,820 INFO [master/hbasemaster-0:16000:becomeActiveMaster] wal.WALProcedureStore: Rolled new Procedure Store WAL, id=2528 2020-05-21 17:26:37,823 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Recovered WALProcedureStore lease in 227msec 2020-05-21 17:26:37,828 INFO [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Rebuilding tracker for hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log 2020-05-21 17:26:37,975 WARN [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Nothing left to decode. Exiting with missing EOF, log=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log 2020-05-21 17:26:37,975 INFO [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Read 2240 entries in hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log 2020-05-21 17:26:38,044 INFO [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Rebuilding tracker for hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:26:38,786 WARN [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Nothing left to decode. Exiting with missing EOF, log=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:26:38,786 INFO [master/hbasemaster-0:16000:becomeActiveMaster] wal.ProcedureWALFormatReader: Read 12854 entries in hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:26:38,996 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure.MasterProcedureScheduler: Took xlock for pid=2497, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, REOPEN/MOVE 2020-05-21 17:26:38,996 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure.MasterProcedureScheduler: Took xlock for pid=2500, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, REOPEN/MOVE 2020-05-21 17:26:38,996 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure.MasterProcedureScheduler: Took xlock for pid=2503, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:26:39,046 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Loaded WALProcedureStore in 1.2220sec 2020-05-21 17:26:39,047 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.RemoteProcedureDispatcher: Instantiated, coreThreads=128 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2020-05-21 17:26:39,093 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.AssignmentManager: Attach pid=2497, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, REOPEN/MOVE to rit=OFFLINE, location=null, table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2 to restore RIT 2020-05-21 17:26:39,094 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.AssignmentManager: Attach pid=2500, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, REOPEN/MOVE to rit=OFFLINE, location=null, table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86 to restore RIT 2020-05-21 17:26:39,094 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.AssignmentManager: Attach pid=2503, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE to rit=OFFLINE, location=null, table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee to restore RIT 2020-05-21 17:26:39,104 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.RegionServerTracker: Starting RegionServerTracker; 0 have existing ServerCrashProcedures, 3 possibly 'live' servers, and 1 'splitting'. 2020-05-21 17:26:39,105 ERROR [master/hbasemaster-0:16000:becomeActiveMaster] master.RegionServerTracker: regionserver-0.hbase.hbase.svc.cluster.local,16020,1585071977140 has no matching ServerCrashProcedure 2020-05-21 17:26:39,173 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Registering regionserver=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:26:39,176 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Registering regionserver=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:26:39,177 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Registering regionserver=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:26:39,265 INFO [master/hbasemaster-0:16000:becomeActiveMaster] balancer.BaseLoadBalancer: slop=0.1, systemTablesOnMaster=false 2020-05-21 17:26:39,276 INFO [master/hbasemaster-0:16000:becomeActiveMaster] balancer.StochasticLoadBalancer: Loaded config; maxSteps=1000000, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, etc. 2020-05-21 17:26:39,349 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.HMaster: Active/primary master=hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313, sessionid=0x10000829d3e2864, setting cluster-up flag (Was=true) 2020-05-21 17:26:39,384 WARN [master/hbasemaster-0:16000:becomeActiveMaster] snapshot.SnapshotManager: Couldn't delete working snapshot directory: hdfs://aeris/hbase/.hbase-snapshot/.tmp 2020-05-21 17:26:39,546 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.HMaster: hbase:meta {1588230740 state=OPEN, ts=1590081999091, server=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754} 2020-05-21 17:26:39,568 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.TimeoutExecutorThread: ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1590082029568 2020-05-21 17:26:39,573 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.DirScanPool: Cleaner pool size is 1 2020-05-21 17:26:39,575 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2020-05-21 17:26:39,576 INFO [master/hbasemaster-0:16000:becomeActiveMaster] zookeeper.RecoverableZooKeeper: Process identifier=replicationLogCleaner connecting to ZooKeeper ensemble=zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 2020-05-21 17:26:39,577 INFO [master/hbasemaster-0:16000:becomeActiveMaster] zookeeper.ZooKeeper: Initiating client connection, connectString=zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@3cbe0354 2020-05-21 17:26:39,578 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2020-05-21 17:26:39,579 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2020-05-21 17:26:39,643 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.LogCleaner: Creating 2 OldWALs cleaner threads 2020-05-21 17:26:39,649 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: LogsCleaner Period: 600000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:39,654 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2020-05-21 17:26:39,658 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2020-05-21 17:26:39,659 INFO [master/hbasemaster-0:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2020-05-21 17:26:39,660 INFO [master/hbasemaster-0:16000:becomeActiveMaster-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Opening socket connection to server zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181. Will not attempt to authenticate using SASL (unknown error) 2020-05-21 17:26:39,660 INFO [master/hbasemaster-0:16000:becomeActiveMaster-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Socket connection established to zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181, initiating session 2020-05-21 17:26:39,663 INFO [master/hbasemaster-0:16000:becomeActiveMaster-SendThread(zookeeper-0-zookeeper.zookeeper.svc.cluster.local:2181)] zookeeper.ClientCnxn: Session establishment complete on server zookeeper-0-zookeeper.zookeeper.svc.cluster.local/172.29.90.3:2181, sessionid = 0x10000829d3e2869, negotiated timeout = 40000 2020-05-21 17:26:39,665 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: HFileCleaner Period: 600000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:39,667 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: ReplicationBarrierCleaner Period: 43200000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:39,668 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.HMaster: Master startup: status=Wait for region servers to report in, state=RUNNING, startTime=1590081401251, completionTime=-1 2020-05-21 17:26:39,668 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=3; waited=0ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=0ms 2020-05-21 17:26:41,172 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=3; waited=1504ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=-1504ms 2020-05-21 17:26:41,709 INFO [RS-EventLoopGroup-1-1] hbase.Server: Connection from 10.128.9.37:50537, version=2.2.4, sasl=false, ugi=hbase (auth:SIMPLE), service=RegionServerStatusService 2020-05-21 17:26:41,709 INFO [RS-EventLoopGroup-1-2] hbase.Server: Connection from 10.128.14.39:44929, version=2.2.4, sasl=false, ugi=hbase (auth:SIMPLE), service=RegionServerStatusService 2020-05-21 17:26:41,953 INFO [RS-EventLoopGroup-1-2] hbase.Server: Connection from 10.128.53.53:44503, version=2.2.4, sasl=false, ugi=hbase (auth:SIMPLE), service=RegionServerStatusService 2020-05-21 17:26:42,677 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=3; waited=3008ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=-3008ms 2020-05-21 17:26:44,182 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.ServerManager: Finished waiting on RegionServer count=3; waited=4514ms, expected min=1 server(s), max=NO_LIMIT server(s), master is running 2020-05-21 17:26:44,355 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=20404052b3c0674a71c9f9da299b9e17, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=240 2020-05-21 17:26:44,356 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=258 2020-05-21 17:26:44,357 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=ba13fae1aa340cfe87b198638663432f, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=249 2020-05-21 17:26:44,357 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=a6bc10ee35b217018a9e0d67385305e9, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=292 2020-05-21 17:26:44,358 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=6c4c2be586f7ad04823bf64879bf863e, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=260 2020-05-21 17:26:44,359 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=5a90663185105f199068aadf7f495f28, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=260 2020-05-21 17:26:44,359 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=73993ad09b31c2a8063646a340a37327, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=253 2020-05-21 17:26:44,360 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=a6d29d67bde8e42b9fe72ebb4df3be68, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=247 2020-05-21 17:26:44,360 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=47a8ba48785bcfd5a3bf576c212eb570, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=240 2020-05-21 17:26:44,361 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=9eda3f5973caeeba814cec96879ff797, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=273 2020-05-21 17:26:44,362 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=521214e7702689731813492a1dc4b54a, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=273 2020-05-21 17:26:44,362 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=895c51e25b71d8edc8d063478d5e959d, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=258 2020-05-21 17:26:44,363 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=7bc9b31cffc00199e00820c96bb73d39, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=244 2020-05-21 17:26:44,363 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=6fc9d3397f8a7edbd4dd190aaef975c3, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=259 2020-05-21 17:26:44,364 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=fb8afdaeaa529390bffddea5ea7a76fd, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=255 2020-05-21 17:26:44,364 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=9edb5377034d812a87d2487858346194, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=248 2020-05-21 17:26:44,365 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=85da959eab50f93627ef35794d04ba36, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=291 2020-05-21 17:26:44,365 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=1f39c96ff0915a836361d227823976f7, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=243 2020-05-21 17:26:44,366 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=6b3928169202b04437a412abcc3c3e84, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=234 2020-05-21 17:26:44,366 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=45b9358748dd408f57d5ee0a533f5934, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=267 2020-05-21 17:26:44,366 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=66abcacc5fe6701f54279e5cc22b758f, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=98434718 2020-05-21 17:26:44,367 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=8a83dfc2fd90e7fc0f6ee8c530093fa2, regionState=OPENING, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, openSeqNum=142833451 2020-05-21 17:26:44,368 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=6d850acf1457152a77813dc3c02d3d86, regionState=OPENING, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, openSeqNum=116806436 2020-05-21 17:26:44,368 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=eb1b85293f32138e11c1885be335155b, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=113655439 2020-05-21 17:26:44,369 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=fbbd176edeb2884b2cab67dbf033f887, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=113014536 2020-05-21 17:26:44,369 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=94642645d0ba481c1c06c18a7b31f122, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=98018523 2020-05-21 17:26:44,369 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=10c81f35e28121d9c1f3f730c673728b, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=98018866 2020-05-21 17:26:44,370 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=2d70a3e092fcb7c31e732610f34e42cb, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=119573494 2020-05-21 17:26:44,370 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=9bf8f6c9ac2972f06898c86000e5f79f, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=109978745 2020-05-21 17:26:44,371 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=86533e2f7328258a6b5448a4b09aee15, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=109023214 2020-05-21 17:26:44,371 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=f4e57ea829d51cdcb1c8d74a938311c2, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=98729325 2020-05-21 17:26:44,372 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=bcfcfdf8cf1e7461eb7353580e638d1a, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=104264129 2020-05-21 17:26:44,372 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=4d52816856fdb4e2a80d432d24b02872, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=115582490 2020-05-21 17:26:44,373 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=789e37c023bf4b11823c10883a42ef94, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=115233622 2020-05-21 17:26:44,373 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=707360501b0ab41efd74025af857bbb7, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=121128146 2020-05-21 17:26:44,374 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=6247174694aa0f08b4248bcfcb7e8b9e, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=98085641 2020-05-21 17:26:44,374 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=fd76870dec2ac0f1e1f8fe0f563b5611, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=100510636 2020-05-21 17:26:44,375 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=e855ef99e11ec78bead3c9e21e6da12f, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=90599701 2020-05-21 17:26:44,375 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=5b21c77606219c0cd30298e8c40566fa, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=105781305 2020-05-21 17:26:44,375 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=595f47d1f2db471835be0c1ce74b91f5, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=111331358 2020-05-21 17:26:44,376 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=a20cc6185462b4c4f9e590d517ad8746, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=94045341 2020-05-21 17:26:44,376 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=cb592da589b3dd3af1fd129c458e2404, regionState=OPEN, lastHost=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, openSeqNum=80144616 2020-05-21 17:26:44,377 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=92440ed8ef80bb8614befd3e89e39326, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=103551154 2020-05-21 17:26:44,377 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=58c894e046698664c5c4b16292cf95ee, regionState=OPENING, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, openSeqNum=100963046 2020-05-21 17:26:44,378 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.RegionStateStore: Load hbase:meta entry region=868e9d680ced53e3a97e0ff466d6aef9, regionState=OPEN, lastHost=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, openSeqNum=313 2020-05-21 17:26:44,378 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.AssignmentManager: Number of RegionServers=3 2020-05-21 17:26:44,379 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.TimeoutExecutorThread: ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1590082064379 2020-05-21 17:26:44,379 INFO [master/hbasemaster-0:16000:becomeActiveMaster] procedure2.TimeoutExecutorThread: ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1590082124379 2020-05-21 17:26:44,379 INFO [master/hbasemaster-0:16000:becomeActiveMaster] assignment.AssignmentManager: Joined the cluster in 196msec 2020-05-21 17:26:44,647 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-ClusterStatusChore Period: 60000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:44,647 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-BalancerChore Period: 300000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:44,649 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-RegionNormalizerChore Period: 300000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:44,664 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: CatalogJanitor-hbasemaster-0:16000 Period: 300000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:44,667 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: HbckChore- Period: 3600000 Unit: MILLISECONDS] is enabled. 2020-05-21 17:26:44,760 WARN [master/hbasemaster-0:16000.Chore.1] master.CatalogJanitor: Playing-it-safe skipping merge/split gc'ing of regions from hbase:meta while regions-in-transition (RIT) 2020-05-21 17:26:44,896 INFO [master/hbasemaster-0:16000:becomeActiveMaster] master.HMaster: Master has completed initialization 8.696sec 2020-05-21 17:26:44,966 INFO [master/hbasemaster-0:16000:becomeActiveMaster] quotas.MasterQuotaManager: Quota support disabled 2020-05-21 17:26:44,967 INFO [master/hbasemaster-0:16000:becomeActiveMaster] zookeeper.ZKWatcher: not a secure deployment, proceeding 2020-05-21 17:26:44,973 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-ExpiredMobFileCleanerChore Period: 86400 Unit: SECONDS] is enabled. 2020-05-21 17:26:44,979 INFO [master/hbasemaster-0:16000:becomeActiveMaster] hbase.ChoreService: Chore [ScheduledChore: Name: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-MobCompactionChore Period: 604800 Unit: SECONDS] is enabled. 2020-05-21 17:26:46,134 INFO [PEWorker-7] assignment.RegionStateStore: pid=2497 updating hbase:meta row=8a83dfc2fd90e7fc0f6ee8c530093fa2, regionState=OPEN, openSeqNum=142836613, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:26:46,134 INFO [PEWorker-9] assignment.RegionStateStore: pid=2500 updating hbase:meta row=6d850acf1457152a77813dc3c02d3d86, regionState=OPEN, openSeqNum=116811181, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:26:46,185 INFO [PEWorker-2] assignment.RegionStateStore: pid=2503 updating hbase:meta row=58c894e046698664c5c4b16292cf95ee, regionState=OPEN, openSeqNum=100966345, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:26:46,246 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished subprocedure pid=2502, resume processing parent pid=2500, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, REOPEN/MOVE 2020-05-21 17:26:46,247 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2502, ppid=2500, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 1mins, 57.592sec 2020-05-21 17:26:46,245 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished subprocedure pid=2499, resume processing parent pid=2497, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, REOPEN/MOVE 2020-05-21 17:26:46,252 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished pid=2499, ppid=2497, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 6mins, 4.573sec 2020-05-21 17:26:46,353 INFO [PEWorker-2] procedure2.ProcedureExecutor: Finished subprocedure pid=2505, resume processing parent pid=2503, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:26:46,353 INFO [PEWorker-2] procedure2.ProcedureExecutor: Finished pid=2505, ppid=2503, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 1mins, 40.769sec 2020-05-21 17:26:46,362 INFO [PEWorker-8] procedure2.ProcedureExecutor: Finished pid=2497, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, REOPEN/MOVE in 6mins, 5.413sec 2020-05-21 17:26:46,362 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished pid=2500, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, REOPEN/MOVE in 1mins, 58.517sec 2020-05-21 17:26:46,470 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished pid=2503, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE in 1mins, 41.968sec 2020-05-21 17:27:44,244 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a] zookeeper.ZooKeeper: Session: 0x200008282832a6b closed 2020-05-21 17:27:44,245 INFO [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x200008282832a6b 2020-05-21 17:28:26,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726581213282221 ns. 2020-05-21 17:28:26,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 313761 ns. 2020-05-21 17:28:26,999 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 500 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4090 bytes 2020-05-21 17:28:27,000 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 500 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082117000 executing as hbase 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,001 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:27,002 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:27,004 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:27,006 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 500 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082117000 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 541 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 5 totalTime: 5 2020-05-21 17:28:27,046 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 158 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 740 bytes 2020-05-21 17:28:27,047 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 158 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082117047 executing as hbase 2020-05-21 17:28:27,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100966346 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=3, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.] 2020-05-21 17:28:27,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 158 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082117047 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 285 } connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:27,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:28:27,094 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 500 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3680 bytes 2020-05-21 17:28:27,094 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 500 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082117094 executing as hbase 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,095 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:27,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ent\xC4\x07counter,1587075435806.4d52816856fdb4e2a80d432d24b02872., aeris_v2,\x0B\xCD\x02-\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436265.789e37c023bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:27,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 500 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082117094 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 629 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:27,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726582213290986 ns. 2020-05-21 17:28:27,257 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 796135 ns. 2020-05-21 17:28:28,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726583213253849 ns. 2020-05-21 17:28:28,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 220800 ns. 2020-05-21 17:28:29,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726584213276689 ns. 2020-05-21 17:28:29,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 180366 ns. 2020-05-21 17:28:29,573 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082109573, delay=0 2020-05-21 17:28:30,009 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 501 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4090 bytes 2020-05-21 17:28:30,009 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 501 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082120009 executing as hbase 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:30,010 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:30,011 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:30,011 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 501 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082120009 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 541 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:30,054 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 159 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 740 bytes 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 159 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082120054 executing as hbase 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100966346 2020-05-21 17:28:30,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=3, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.] 2020-05-21 17:28:30,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 159 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082120054 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 285 } connection: 10.128.14.39:44929, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:28:30,099 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 501 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3680 bytes 2020-05-21 17:28:30,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 501 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082120100 executing as hbase 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,102 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,103 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:30,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:30,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:30,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ent\xC4\x07counter,1587075435806.4d52816856fdb4e2a80d432d24b02872., aeris_v2,\x0B\xCD\x02-\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436265.789e37c023bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:30,105 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 501 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082120100 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 629 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 4 totalTime: 4 2020-05-21 17:28:30,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726585213265729 ns. 2020-05-21 17:28:30,257 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 860395 ns. 2020-05-21 17:28:31,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726586213284720 ns. 2020-05-21 17:28:31,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 218743 ns. 2020-05-21 17:28:32,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726587213329775 ns. 2020-05-21 17:28:32,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 239029 ns. 2020-05-21 17:28:33,014 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 502 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4090 bytes 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 502 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082123014 executing as hbase 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,015 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:33,016 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:33,017 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 502 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082123014 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 405 } connection: 10.128.9.37:50537, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:28:33,062 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 160 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 740 bytes 2020-05-21 17:28:33,063 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 160 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082123063 executing as hbase 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100966346 2020-05-21 17:28:33,064 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=3, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.] 2020-05-21 17:28:33,065 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 160 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082123063 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 204 } connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:33,107 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 502 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3680 bytes 2020-05-21 17:28:33,108 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 502 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082123108 executing as hbase 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:33,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ent\xC4\x07counter,1587075435806.4d52816856fdb4e2a80d432d24b02872., aeris_v2,\x0B\xCD\x02-\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436265.789e37c023bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:33,110 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 502 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082123108 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 444 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:33,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726588213220208 ns. 2020-05-21 17:28:33,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 626445 ns. 2020-05-21 17:28:34,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726589213346441 ns. 2020-05-21 17:28:34,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 269135 ns. 2020-05-21 17:28:34,573 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082114573, delay=0 2020-05-21 17:28:35,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726590213364587 ns. 2020-05-21 17:28:35,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 176358 ns. 2020-05-21 17:28:36,020 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 503 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4090 bytes 2020-05-21 17:28:36,020 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 503 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082126020 executing as hbase 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:36,021 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:36,022 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 503 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082126020 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 405 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:36,070 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 161 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 740 bytes 2020-05-21 17:28:36,071 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 161 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082126071 executing as hbase 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100966346 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=3, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.] 2020-05-21 17:28:36,072 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 161 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082126071 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 249 } connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:36,112 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 503 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3680 bytes 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 503 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082126113 executing as hbase 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:36,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:36,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:36,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ent\xC4\x07counter,1587075435806.4d52816856fdb4e2a80d432d24b02872., aeris_v2,\x0B\xCD\x02-\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436265.789e37c023bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:36,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 503 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082126113 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 444 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:36,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726591213324541 ns. 2020-05-21 17:28:36,257 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 855906 ns. 2020-05-21 17:28:37,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:28:37,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726592213260496 ns. 2020-05-21 17:28:37,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 106402 ns. 2020-05-21 17:28:37,846 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcServer: Connection /10.128.25.30:47836; # active connections=3 2020-05-21 17:28:37,847 INFO [RS-EventLoopGroup-1-2] hbase.Server: Connection from 10.128.25.30:47836, version=2.2.4, sasl=false, ugi=hbase (auth:SIMPLE), service=MasterService 2020-05-21 17:28:37,847 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 17 method_name: "IsMasterRunning" request_param: true priority: 0 timeout: 60000 totalRequestSize: 29 bytes 2020-05-21 17:28:37,850 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] ipc.RpcServer: callId: 17 service: MasterService methodName: IsMasterRunning size: 29 connection: 10.128.25.30:47836 deadline: 1590082177849 executing as hbase 2020-05-21 17:28:37,854 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] ipc.RpcServer: callId: 17 service: MasterService methodName: IsMasterRunning size: 29 connection: 10.128.25.30:47836 deadline: 1590082177849 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$IsMasterRunningRequest connection: 10.128.25.30:47836, response is_master_running: true queueTime: 1 processingTime: 3 totalTime: 4 2020-05-21 17:28:37,858 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 18 method_name: "Balance" request_param: true priority: 0 timeout: 60000 totalRequestSize: 23 bytes 2020-05-21 17:28:37,860 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] ipc.RpcServer: callId: 18 service: MasterService methodName: Balance size: 23 connection: 10.128.25.30:47836 deadline: 1590082177860 executing as hbase 2020-05-21 17:28:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: PrimaryRegionCountSkewCostFunction not needed 2020-05-21 17:28:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: RegionReplicaHostCostFunction not needed 2020-05-21 17:28:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: RegionReplicaRackCostFunction not needed 2020-05-21 17:28:37,888 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: start StochasticLoadBalancer.balancer, initCost=215.89316106477503, functionCost=RegionCountSkewCostFunction : (500.0, 0.4); PrimaryRegionCountSkewCostFunction : (500.0, 0.0); MoveCostFunction : (7.0, 0.0); ServerLocalityCostFunction : (25.0, 0.0); RackLocalityCostFunction : (15.0, 0.0); TableSkewCostFunction : (35.0, 0.3); RegionReplicaHostCostFunction : (100000.0, 0.0); RegionReplicaRackCostFunction : (10000.0, 0.0); ReadRequestCostFunction : (5.0, 0.1002114661654135); WriteRequestCostFunction : (5.0, 0.3994591489302078); MemStoreSizeCostFunction : (5.0, 0.4507772020725389); StoreFileCostFunction : (5.0, 0.12818439578684088); computedMaxSteps: 108000 2020-05-21 17:28:38,266 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726593223322502 ns. 2020-05-21 17:28:38,266 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 319687 ns. 2020-05-21 17:28:39,024 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 504 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4090 bytes 2020-05-21 17:28:39,044 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 504 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082129044 executing as hbase 2020-05-21 17:28:39,045 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,046 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,046 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,048 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:39,049 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:39,050 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:39,051 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 504 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082129044 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 348 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 7 totalTime: 7 2020-05-21 17:28:39,086 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 162 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 740 bytes 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 162 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082129086 executing as hbase 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100966346 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=3, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.] 2020-05-21 17:28:39,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 162 service: RegionServerStatusService methodName: RegionServerReport size: 740 connection: 10.128.14.39:44929 deadline: 1590082129086 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 249 } connection: 10.128.14.39:44929, response queueTime: 1 processingTime: 0 totalTime: 1 2020-05-21 17:28:39,118 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 504 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3680 bytes 2020-05-21 17:28:39,144 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 504 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082129144 executing as hbase 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:39,145 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ent\xC4\x07counter,1587075435806.4d52816856fdb4e2a80d432d24b02872., aeris_v2,\x0B\xCD\x02-\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436265.789e37c023bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:39,146 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 504 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.53.53:44503 deadline: 1590082129144 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 464 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:39,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726594213231231 ns. 2020-05-21 17:28:39,257 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 809000 ns. 2020-05-21 17:28:39,562 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 58c894e046698664c5c4b16292cf95ee from server regionserver-2.hbase.hbase.svc.cluster.local to regionserver-0.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 9eda3f5973caeeba814cec96879ff797 from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 82ccc4215fd1fe7a2a329a0c6c297fa4 from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region ba13fae1aa340cfe87b198638663432f from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region cb592da589b3dd3af1fd129c458e2404 from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 20404052b3c0674a71c9f9da299b9e17 from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region a6bc10ee35b217018a9e0d67385305e9 from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 10c81f35e28121d9c1f3f730c673728b from server regionserver-1.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region f4e57ea829d51cdcb1c8d74a938311c2 from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region e855ef99e11ec78bead3c9e21e6da12f from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 6fc9d3397f8a7edbd4dd190aaef975c3 from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 707360501b0ab41efd74025af857bbb7 from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region 73993ad09b31c2a8063646a340a37327 from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Moving Region fbbd176edeb2884b2cab67dbf033f887 from server regionserver-0.hbase.hbase.svc.cluster.local to regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:39,563 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] balancer.StochasticLoadBalancer: Finished computing new load balance plan. Computation took PT1.678S to try 108000 different iterations. Found a solution that moves 14 regions; Going from a computed cost of 215.89316106477503 to a new cost of 4.396304067639544 2020-05-21 17:28:39,564 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: Balancer plans size is 14, the balance interval is 21428 ms, and the max number regions in transition is 46 2020-05-21 17:28:39,564 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=58c894e046698664c5c4b16292cf95ee, source=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, destination=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:39,566 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE, subproc=null 2020-05-21 17:28:39,573 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082119573, delay=0 2020-05-21 17:28:39,668 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=14.1K (118/sec) 2020-05-21 17:28:39,747 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=467 2020-05-21 17:28:39,747 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=14.5K (122/sec) 2020-05-21 17:28:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,752 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,752 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2507 runningCount=1, activeCount=1 2020-05-21 17:28:39,752 INFO [PEWorker-12] procedure.MasterProcedureScheduler: Took xlock for pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,752 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,761 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=470 2020-05-21 17:28:39,761 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=15.0K (126/sec) 2020-05-21 17:28:39,761 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; cycles=0 2020-05-21 17:28:39,761 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:39,762 INFO [PEWorker-12] assignment.RegionStateStore: pid=2507 updating hbase:meta row=58c894e046698664c5c4b16292cf95ee, regionState=CLOSING, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:39,769 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 15, totalSize: 6 bytes 2020-05-21 17:28:39,769 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 7ms 2020-05-21 17:28:39,770 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:39,771 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:39,771 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:39,771 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE, children [pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:39,771 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE, subproc=[pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:39,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=961 2020-05-21 17:28:39,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=15.9K (134/sec) 2020-05-21 17:28:39,780 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,780 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,780 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2508 runningCount=2, activeCount=2 2020-05-21 17:28:39,780 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2507 runningCount=1, activeCount=1 2020-05-21 17:28:39,782 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,791 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=490 2020-05-21 17:28:39,792 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=16.4K (138/sec) 2020-05-21 17:28:39,792 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=false, [] 2020-05-21 17:28:39,792 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,792 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:39,793 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,793 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,801 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=492 2020-05-21 17:28:39,802 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=16.9K (142/sec) 2020-05-21 17:28:39,802 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:39,809 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=489 2020-05-21 17:28:39,809 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=17.4K (146/sec) 2020-05-21 17:28:39,809 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2508 runningCount=16, activeCount=0 2020-05-21 17:28:39,944 TRACE [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:39,955 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 16, totalSize: 4 bytes 2020-05-21 17:28:39,955 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 8ms 2020-05-21 17:28:40,004 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner, timeout=1590082120004, delay=0 2020-05-21 17:28:40,005 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2185, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=868e9d680ced53e3a97e0ff466d6aef9, REOPEN/MOVE 2020-05-21 17:28:40,006 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2188, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=45b9358748dd408f57d5ee0a533f5934, REOPEN/MOVE 2020-05-21 17:28:40,007 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2191, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, REOPEN/MOVE 2020-05-21 17:28:40,007 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2204, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=a20cc6185462b4c4f9e590d517ad8746, REOPEN/MOVE 2020-05-21 17:28:40,008 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2207, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:40,008 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2210, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=eb1b85293f32138e11c1885be335155b, REOPEN/MOVE 2020-05-21 17:28:40,009 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2213, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=5b21c77606219c0cd30298e8c40566fa, REOPEN/MOVE 2020-05-21 17:28:40,009 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2216, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=895c51e25b71d8edc8d063478d5e959d, REOPEN/MOVE 2020-05-21 17:28:40,010 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2219, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=fd76870dec2ac0f1e1f8fe0f563b5611, REOPEN/MOVE 2020-05-21 17:28:40,010 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2222, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:40,010 TRACE [ProcExecTimeout] wal.WALProcedureStore: Delete [2185, 2188, 2191, 2204, 2207, 2210, 2213, 2216, 2219, 2222] 2020-05-21 17:28:40,111 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=17.4K (146/sec) 2020-05-21 17:28:40,118 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=60 2020-05-21 17:28:40,118 INFO [WALProcedureStoreSyncThread] wal.WALProcedureStore: Remove the oldest log hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:28:40,118 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Removing log=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log 2020-05-21 17:28:40,118 INFO [WALProcedureStoreSyncThread] wal.ProcedureWALFile: Archiving hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log to hdfs://aeris/hbase/oldWALs/pv2-00000000000000002526.log 2020-05-21 17:28:40,129 DEBUG [WALProcedureStoreSyncThread] wal.WALProcedureStore: Removed log=hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002526.log, activeLogs=[hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002527.log, hdfs://aeris/hbase/MasterProcWALs/pv2-00000000000000002528.log] 2020-05-21 17:28:40,129 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=17.4K (146/sec) 2020-05-21 17:28:40,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726595213237109 ns. 2020-05-21 17:28:40,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 136888 ns. 2020-05-21 17:28:40,736 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 163 method_name: "ReportFileArchival" request_param: true priority: 0 timeout: 10000 totalRequestSize: 217 bytes 2020-05-21 17:28:40,741 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 163 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.14.39:44929 deadline: 1590082130741 executing as hbase 2020-05-21 17:28:40,743 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 163 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.14.39:44929 deadline: 1590082130741 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$FileArchiveNotificationRequest connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:40,799 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 164 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 242 bytes 2020-05-21 17:28:40,799 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 164 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 242 connection: 10.128.14.39:44929 deadline: 1590082130799 executing as hbase 2020-05-21 17:28:40,800 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 region=rit=CLOSING, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee regionState=CLOSED 2020-05-21 17:28:40,800 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,901 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=17.4K (145/sec) 2020-05-21 17:28:40,908 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=502 2020-05-21 17:28:40,908 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=17.9K (149/sec) 2020-05-21 17:28:40,909 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=true, [pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:40,909 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,909 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,910 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:40,910 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2508, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,910 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2508 runningCount=1, activeCount=1 2020-05-21 17:28:40,910 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,912 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 164 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 242 connection: 10.128.14.39:44929 deadline: 1590082130799 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 111 totalTime: 111 2020-05-21 17:28:40,917 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=505 2020-05-21 17:28:40,917 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=18.4K (153/sec) 2020-05-21 17:28:40,917 INFO [PEWorker-13] assignment.RegionStateStore: pid=2507 updating hbase:meta row=58c894e046698664c5c4b16292cf95ee, regionState=CLOSED 2020-05-21 17:28:40,920 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 17, totalSize: 6 bytes 2020-05-21 17:28:40,920 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:40,921 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2508, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:40,921 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2508, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,921 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,927 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=507 2020-05-21 17:28:40,927 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=18.9K (157/sec) 2020-05-21 17:28:40,927 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2508, ppid=2507, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:40,933 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=504 2020-05-21 17:28:40,933 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=19.4K (161/sec) 2020-05-21 17:28:40,934 TRACE [PEWorker-13] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:40,934 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,940 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=478 2020-05-21 17:28:40,941 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=19.9K (165/sec) 2020-05-21 17:28:40,942 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE has lock 2020-05-21 17:28:40,942 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,942 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2507 runningCount=2, activeCount=2 2020-05-21 17:28:40,942 TRACE [PEWorker-14] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; cycles=0 2020-05-21 17:28:40,942 TRACE [PEWorker-14] procedure2.StateMachineProcedure: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,942 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2507 2020-05-21 17:28:40,942 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:40,942 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,942 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,943 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished subprocedure pid=2508, resume processing parent pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:40,943 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2508, ppid=2507, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 1.1510sec 2020-05-21 17:28:40,943 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2508 runningCount=1, activeCount=1 2020-05-21 17:28:41,043 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=19.9K (165/sec) 2020-05-21 17:28:41,051 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/2, flushed=482 2020-05-21 17:28:41,051 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=20.3K (169/sec) 2020-05-21 17:28:41,051 TRACE [PEWorker-14] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; cycles=0 2020-05-21 17:28:41,051 TRACE [PEWorker-14] procedure2.StateMachineProcedure: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,052 INFO [PEWorker-14] assignment.TransitRegionStateProcedure: Starting pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066; forceNewPlan=false, retain=false 2020-05-21 17:28:41,052 TRACE [PEWorker-14] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=false, [] 2020-05-21 17:28:41,052 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Suspend pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,052 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:41,052 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,052 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,058 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=486 2020-05-21 17:28:41,059 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=20.8K (173/sec) 2020-05-21 17:28:41,059 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2507 runningCount=16, activeCount=0 2020-05-21 17:28:41,202 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:41,203 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=3: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:41,203 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066} 2020-05-21 17:28:41,206 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment. 2020-05-21 17:28:41,207 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=true, [pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE] 2020-05-21 17:28:41,207 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,208 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE has lock 2020-05-21 17:28:41,208 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 1msec 2020-05-21 17:28:41,209 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,209 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2507 runningCount=1, activeCount=1 2020-05-21 17:28:41,209 TRACE [PEWorker-15] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; cycles=0 2020-05-21 17:28:41,209 TRACE [PEWorker-15] procedure2.StateMachineProcedure: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,209 INFO [PEWorker-15] assignment.RegionStateStore: pid=2507 updating hbase:meta row=58c894e046698664c5c4b16292cf95ee, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:41,213 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 18, totalSize: 6 bytes 2020-05-21 17:28:41,213 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:41,214 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:41,214 INFO [PEWorker-15] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:41,214 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:41,214 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Stored pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE, children [pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:41,214 TRACE [PEWorker-15] wal.WALProcedureStore: Insert pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE, subproc=[pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:41,221 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=911 2020-05-21 17:28:41,222 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=21.7K (180/sec) 2020-05-21 17:28:41,222 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,222 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,222 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2509 runningCount=2, activeCount=2 2020-05-21 17:28:41,223 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,223 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2507 runningCount=1, activeCount=1 2020-05-21 17:28:41,233 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=428 2020-05-21 17:28:41,234 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=22.1K (184/sec) 2020-05-21 17:28:41,234 TRACE [PEWorker-16] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=false, [] 2020-05-21 17:28:41,234 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Suspend pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,234 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:41,234 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Store update pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,234 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,240 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:41,240 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=22.6K (187/sec) 2020-05-21 17:28:41,240 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,246 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=427 2020-05-21 17:28:41,246 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=23.0K (191/sec) 2020-05-21 17:28:41,247 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2509 runningCount=16, activeCount=0 2020-05-21 17:28:41,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726596213223826 ns. 2020-05-21 17:28:41,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 118426 ns. 2020-05-21 17:28:41,385 TRACE [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:41,386 DEBUG [RSProcedureDispatcher-pool3-t3] master.ServerManager: New admin connection to regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:41,387 DEBUG [RSProcedureDispatcher-pool3-t3] ipc.RpcConnection: Use SIMPLE authentication for service AdminService, sasl=false 2020-05-21 17:28:41,388 TRACE [RSProcedureDispatcher-pool3-t3] ipc.NettyRpcConnection: Connecting to regionserver-0.hbase.hbase.svc.cluster.local/10.128.53.53:16020 2020-05-21 17:28:41,404 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 19, totalSize: 4 bytes 2020-05-21 17:28:41,404 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 17ms 2020-05-21 17:28:41,482 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 505 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 247 bytes 2020-05-21 17:28:41,483 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 505 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 247 connection: 10.128.53.53:44503 deadline: 1590082131483 executing as hbase 2020-05-21 17:28:41,483 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee regionState=OPENED 2020-05-21 17:28:41,483 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,584 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=23.0K (190/sec) 2020-05-21 17:28:41,591 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=434 2020-05-21 17:28:41,591 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=23.4K (194/sec) 2020-05-21 17:28:41,592 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 58c894e046698664c5c4b16292cf95ee, NAME => 'aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee.', STARTKEY => '\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m', ENDKEY => ''}, ready=true, [pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:41,592 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,592 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,592 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 505 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 247 connection: 10.128.53.53:44503 deadline: 1590082131483 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 109 totalTime: 109 2020-05-21 17:28:41,592 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2509, ppid=2507, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,593 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2509 runningCount=1, activeCount=1 2020-05-21 17:28:41,593 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,600 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=437 2020-05-21 17:28:41,600 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=23.8K (197/sec) 2020-05-21 17:28:41,600 INFO [PEWorker-11] assignment.RegionStateStore: pid=2507 updating hbase:meta row=58c894e046698664c5c4b16292cf95ee, regionState=OPEN, openSeqNum=100968485, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:41,603 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 20, totalSize: 6 bytes 2020-05-21 17:28:41,603 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:41,604 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2509, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:41,604 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Store update pid=2509, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,604 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,612 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=439 2020-05-21 17:28:41,612 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=24.2K (201/sec) 2020-05-21 17:28:41,612 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2509, ppid=2507, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:41,618 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=436 2020-05-21 17:28:41,618 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=24.7K (204/sec) 2020-05-21 17:28:41,618 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2507, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:41,618 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,624 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=490 2020-05-21 17:28:41,624 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=25.2K (208/sec) 2020-05-21 17:28:41,624 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE has lock 2020-05-21 17:28:41,624 INFO [PEWorker-11] procedure2.ProcedureExecutor: Finished subprocedure pid=2509, resume processing parent pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,624 INFO [PEWorker-11] procedure2.ProcedureExecutor: Finished pid=2509, ppid=2507, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 390msec 2020-05-21 17:28:41,624 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2509 runningCount=16, activeCount=0 2020-05-21 17:28:41,624 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,625 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2507 runningCount=1, activeCount=1 2020-05-21 17:28:41,625 TRACE [PEWorker-1] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE; cycles=0 2020-05-21 17:28:41,625 TRACE [PEWorker-1] procedure2.StateMachineProcedure: pid=2507, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,625 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2507, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:41,625 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2507, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,625 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2507, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE and Delete [2508, 2509] 2020-05-21 17:28:41,631 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=507 2020-05-21 17:28:41,631 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=25.6K (212/sec) 2020-05-21 17:28:41,631 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2507, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE 2020-05-21 17:28:41,638 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=495 2020-05-21 17:28:41,638 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=26.1K (216/sec) 2020-05-21 17:28:41,638 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2507, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE released the shared lock 2020-05-21 17:28:41,638 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished pid=2507, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=58c894e046698664c5c4b16292cf95ee, REOPEN/MOVE in 2.0590sec 2020-05-21 17:28:41,638 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2507 runningCount=16, activeCount=0 2020-05-21 17:28:41,654 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=9eda3f5973caeeba814cec96879ff797, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:41,655 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE, subproc=null 2020-05-21 17:28:41,755 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=26.1K (216/sec) 2020-05-21 17:28:41,761 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:41,762 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=26.5K (219/sec) 2020-05-21 17:28:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,763 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,763 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2510 runningCount=1, activeCount=1 2020-05-21 17:28:41,763 INFO [PEWorker-5] procedure.MasterProcedureScheduler: Took xlock for pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,763 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,769 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:41,770 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=26.9K (222/sec) 2020-05-21 17:28:41,770 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; cycles=0 2020-05-21 17:28:41,770 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:41,770 INFO [PEWorker-5] assignment.RegionStateStore: pid=2510 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:41,774 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 21, totalSize: 6 bytes 2020-05-21 17:28:41,774 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:41,775 TRACE [PEWorker-5] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:41,776 INFO [PEWorker-5] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:41,776 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:41,776 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Stored pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE, children [pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:41,776 TRACE [PEWorker-5] wal.WALProcedureStore: Insert pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE, subproc=[pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:41,781 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:41,781 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=27.7K (229/sec) 2020-05-21 17:28:41,782 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,782 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,782 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2511 runningCount=2, activeCount=2 2020-05-21 17:28:41,782 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2510 runningCount=1, activeCount=1 2020-05-21 17:28:41,782 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,789 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:41,789 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=28.1K (232/sec) 2020-05-21 17:28:41,789 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=false, [] 2020-05-21 17:28:41,789 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,789 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:41,789 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,789 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,794 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:41,794 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=28.5K (235/sec) 2020-05-21 17:28:41,794 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:41,799 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:41,799 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=28.9K (239/sec) 2020-05-21 17:28:41,799 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2511 runningCount=16, activeCount=0 2020-05-21 17:28:41,940 TRACE [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:41,941 DEBUG [RSProcedureDispatcher-pool3-t4] master.ServerManager: New admin connection to regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:41,941 DEBUG [RSProcedureDispatcher-pool3-t4] ipc.RpcConnection: Use SIMPLE authentication for service AdminService, sasl=false 2020-05-21 17:28:41,942 TRACE [RSProcedureDispatcher-pool3-t4] ipc.NettyRpcConnection: Connecting to regionserver-1.hbase.hbase.svc.cluster.local/10.128.9.37:16020 2020-05-21 17:28:41,948 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 22, totalSize: 4 bytes 2020-05-21 17:28:41,949 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 8ms 2020-05-21 17:28:41,976 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 505 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:41,976 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 505 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082131976 executing as hbase 2020-05-21 17:28:41,977 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797 regionState=CLOSED 2020-05-21 17:28:41,977 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,054 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 506 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3940 bytes 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 506 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.9.37:50537 deadline: 1590082132054 executing as hbase 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,055 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:42,056 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:42,058 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 506 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.9.37:50537 deadline: 1590082132054 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 447 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:42,078 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=28.9K (238/sec) 2020-05-21 17:28:42,084 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:42,084 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=29.3K (242/sec) 2020-05-21 17:28:42,085 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=true, [pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:42,085 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,086 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,086 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:42,086 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2511, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,087 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2511 runningCount=1, activeCount=1 2020-05-21 17:28:42,087 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 505 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082131976 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 111 totalTime: 111 2020-05-21 17:28:42,097 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 165 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 506 bytes 2020-05-21 17:28:42,097 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 165 service: RegionServerStatusService methodName: RegionServerReport size: 506 connection: 10.128.14.39:44929 deadline: 1590082132097 executing as hbase 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142836614 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116811182 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 regionCount=2, metaLoaded=true [aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2., aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.] 2020-05-21 17:28:42,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 165 service: RegionServerStatusService methodName: RegionServerReport size: 506 connection: 10.128.14.39:44929 deadline: 1590082132097 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081637367 load { numberOfRequests: 178 } connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:42,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:42,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=29.7K (245/sec) 2020-05-21 17:28:42,105 INFO [PEWorker-9] assignment.RegionStateStore: pid=2510 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=CLOSED 2020-05-21 17:28:42,107 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 23, totalSize: 6 bytes 2020-05-21 17:28:42,107 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:42,108 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2511, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:42,108 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2511, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,108 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,114 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:42,114 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=30.2K (248/sec) 2020-05-21 17:28:42,114 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2511, ppid=2510, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,120 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:42,120 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=30.6K (252/sec) 2020-05-21 17:28:42,120 TRACE [PEWorker-9] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:42,120 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,126 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:42,126 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=31.0K (255/sec) 2020-05-21 17:28:42,126 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE has lock 2020-05-21 17:28:42,126 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished subprocedure pid=2511, resume processing parent pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,126 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2511, ppid=2510, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 333msec 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2511 runningCount=16, activeCount=0 2020-05-21 17:28:42,127 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2510 runningCount=1, activeCount=1 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; cycles=0 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.StateMachineProcedure: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2510 2020-05-21 17:28:42,127 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:42,127 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,127 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,133 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:42,133 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=31.4K (258/sec) 2020-05-21 17:28:42,133 TRACE [PEWorker-9] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; cycles=0 2020-05-21 17:28:42,133 TRACE [PEWorker-9] procedure2.StateMachineProcedure: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,133 INFO [PEWorker-9] assignment.TransitRegionStateProcedure: Starting pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:42,133 TRACE [PEWorker-9] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=false, [] 2020-05-21 17:28:42,133 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Suspend pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,133 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:42,133 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,133 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,140 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:42,140 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=31.8K (262/sec) 2020-05-21 17:28:42,140 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2510 runningCount=16, activeCount=0 2020-05-21 17:28:42,148 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 506 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3918 bytes 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 506 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.53.53:44503 deadline: 1590082132149 executing as hbase 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100966346, completeSequenceId=100968486 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,149 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:42,150 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:42,150 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 506 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.53.53:44503 deadline: 1590082132149 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 528 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:42,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726597213242515 ns. 2020-05-21 17:28:42,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 668178 ns. 2020-05-21 17:28:42,283 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:42,283 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=3: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:42,284 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:42,284 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment. 2020-05-21 17:28:42,285 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=true, [pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE] 2020-05-21 17:28:42,285 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,285 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE has lock 2020-05-21 17:28:42,285 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:42,285 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,285 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2510 runningCount=1, activeCount=1 2020-05-21 17:28:42,285 TRACE [PEWorker-2] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; cycles=0 2020-05-21 17:28:42,285 TRACE [PEWorker-2] procedure2.StateMachineProcedure: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,286 INFO [PEWorker-2] assignment.RegionStateStore: pid=2510 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=OPENING, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:42,288 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 24, totalSize: 6 bytes 2020-05-21 17:28:42,288 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:42,289 TRACE [PEWorker-2] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:42,290 INFO [PEWorker-2] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:42,290 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:42,290 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Stored pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE, children [pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:42,290 TRACE [PEWorker-2] wal.WALProcedureStore: Insert pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE, subproc=[pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:42,302 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:42,302 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=32.5K (267/sec) 2020-05-21 17:28:42,302 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,302 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2510 runningCount=16, activeCount=0 2020-05-21 17:28:42,302 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,302 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2512 runningCount=1, activeCount=1 2020-05-21 17:28:42,302 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,308 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:42,308 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=32.9K (270/sec) 2020-05-21 17:28:42,309 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=false, [] 2020-05-21 17:28:42,309 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,309 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:42,309 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,309 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,314 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:42,314 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=33.2K (273/sec) 2020-05-21 17:28:42,315 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,320 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:42,320 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=33.5K (276/sec) 2020-05-21 17:28:42,320 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2512 runningCount=16, activeCount=0 2020-05-21 17:28:42,459 TRACE [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:42,472 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 25, totalSize: 4 bytes 2020-05-21 17:28:42,472 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 12ms 2020-05-21 17:28:42,515 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 166 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:42,516 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 166 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.14.39:44929 deadline: 1590082132515 executing as hbase 2020-05-21 17:28:42,516 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 region=rit=OPENING, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797 regionState=OPENED 2020-05-21 17:28:42,516 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,617 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=33.5K (275/sec) 2020-05-21 17:28:42,623 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=356 2020-05-21 17:28:42,623 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=33.9K (278/sec) 2020-05-21 17:28:42,624 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=true, [pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:42,624 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,624 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,624 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 166 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.14.39:44929 deadline: 1590082132515 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 109 totalTime: 109 2020-05-21 17:28:42,624 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2512, ppid=2510, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,624 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2512 runningCount=1, activeCount=1 2020-05-21 17:28:42,624 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,630 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=359 2020-05-21 17:28:42,630 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=34.2K (281/sec) 2020-05-21 17:28:42,631 INFO [PEWorker-3] assignment.RegionStateStore: pid=2510 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=OPEN, openSeqNum=276, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:42,633 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 26, totalSize: 6 bytes 2020-05-21 17:28:42,633 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:42,634 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2512, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:42,634 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2512, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,634 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,640 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=361 2020-05-21 17:28:42,641 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=34.6K (284/sec) 2020-05-21 17:28:42,641 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2512, ppid=2510, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:42,647 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=358 2020-05-21 17:28:42,647 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=34.9K (287/sec) 2020-05-21 17:28:42,647 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2510, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:42,648 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,653 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:42,653 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=35.3K (290/sec) 2020-05-21 17:28:42,653 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE has lock 2020-05-21 17:28:42,653 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished subprocedure pid=2512, resume processing parent pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,653 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished pid=2512, ppid=2510, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 345msec 2020-05-21 17:28:42,653 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2512 runningCount=16, activeCount=0 2020-05-21 17:28:42,653 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,653 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2510 runningCount=1, activeCount=1 2020-05-21 17:28:42,653 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE; cycles=0 2020-05-21 17:28:42,653 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2510, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,654 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2510, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:42,654 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2510, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,654 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2510, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE and Delete [2511, 2512] 2020-05-21 17:28:42,659 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:42,659 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=35.8K (293/sec) 2020-05-21 17:28:42,659 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2510, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE 2020-05-21 17:28:42,663 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=82ccc4215fd1fe7a2a329a0c6c297fa4, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:42,663 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE, subproc=null 2020-05-21 17:28:42,666 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=419 2020-05-21 17:28:42,666 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=36.2K (297/sec) 2020-05-21 17:28:42,666 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2510, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE released the shared lock 2020-05-21 17:28:42,666 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished pid=2510, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, REOPEN/MOVE in 998msec 2020-05-21 17:28:42,666 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2510 runningCount=16, activeCount=0 2020-05-21 17:28:42,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=36.2K (296/sec) 2020-05-21 17:28:42,772 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:42,772 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=36.6K (300/sec) 2020-05-21 17:28:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,773 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,774 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2513 runningCount=1, activeCount=1 2020-05-21 17:28:42,774 INFO [PEWorker-12] procedure.MasterProcedureScheduler: Took xlock for pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,774 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,779 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:42,779 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=36.9K (303/sec) 2020-05-21 17:28:42,779 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; cycles=0 2020-05-21 17:28:42,779 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:42,779 INFO [PEWorker-12] assignment.RegionStateStore: pid=2513 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:42,782 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 27, totalSize: 6 bytes 2020-05-21 17:28:42,782 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:42,783 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:42,783 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:42,783 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:42,783 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE, children [pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:42,784 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE, subproc=[pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:42,789 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:42,789 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=37.7K (309/sec) 2020-05-21 17:28:42,789 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,789 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2513 runningCount=16, activeCount=0 2020-05-21 17:28:42,789 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,789 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2514 runningCount=1, activeCount=1 2020-05-21 17:28:42,791 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,796 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:42,796 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=38.1K (312/sec) 2020-05-21 17:28:42,796 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=false, [] 2020-05-21 17:28:42,796 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,796 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:42,796 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,796 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,802 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:42,803 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=38.5K (316/sec) 2020-05-21 17:28:42,803 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:42,807 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:42,808 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=38.9K (319/sec) 2020-05-21 17:28:42,808 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2514 runningCount=16, activeCount=0 2020-05-21 17:28:42,947 TRACE [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:42,950 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 28, totalSize: 4 bytes 2020-05-21 17:28:42,950 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 3ms 2020-05-21 17:28:42,969 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 507 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:42,969 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 507 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082132969 executing as hbase 2020-05-21 17:28:42,970 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4 regionState=CLOSED 2020-05-21 17:28:42,970 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,071 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=38.9K (318/sec) 2020-05-21 17:28:43,076 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:43,077 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=39.4K (322/sec) 2020-05-21 17:28:43,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=true, [pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:43,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,078 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:43,078 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2514, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,078 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2514 runningCount=1, activeCount=1 2020-05-21 17:28:43,078 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 507 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082132969 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 109 totalTime: 109 2020-05-21 17:28:43,087 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:43,087 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=39.8K (325/sec) 2020-05-21 17:28:43,087 INFO [PEWorker-13] assignment.RegionStateStore: pid=2513 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=CLOSED 2020-05-21 17:28:43,090 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 29, totalSize: 6 bytes 2020-05-21 17:28:43,091 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:43,091 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2514, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:43,091 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2514, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,091 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,100 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:43,100 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=40.2K (329/sec) 2020-05-21 17:28:43,101 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2514, ppid=2513, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,106 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:43,106 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=40.6K (332/sec) 2020-05-21 17:28:43,107 TRACE [PEWorker-13] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:43,108 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,116 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:43,116 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=41.0K (335/sec) 2020-05-21 17:28:43,116 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE has lock 2020-05-21 17:28:43,116 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished subprocedure pid=2514, resume processing parent pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,116 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2514, ppid=2513, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 308msec 2020-05-21 17:28:43,116 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2514 runningCount=16, activeCount=0 2020-05-21 17:28:43,116 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,117 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2513 runningCount=1, activeCount=1 2020-05-21 17:28:43,117 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; cycles=0 2020-05-21 17:28:43,117 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,117 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2513 2020-05-21 17:28:43,117 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:43,117 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,117 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,127 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:43,127 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=41.4K (338/sec) 2020-05-21 17:28:43,127 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; cycles=0 2020-05-21 17:28:43,127 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,127 INFO [PEWorker-13] assignment.TransitRegionStateProcedure: Starting pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:43,128 TRACE [PEWorker-13] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=false, [] 2020-05-21 17:28:43,128 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Suspend pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,128 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:43,128 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,128 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,137 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:43,137 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=41.8K (342/sec) 2020-05-21 17:28:43,137 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2513 runningCount=16, activeCount=0 2020-05-21 17:28:43,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726598213228816 ns. 2020-05-21 17:28:43,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 139453 ns. 2020-05-21 17:28:43,278 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:43,278 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=3: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:43,278 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:43,279 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment. 2020-05-21 17:28:43,279 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=true, [pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE] 2020-05-21 17:28:43,279 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,279 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE has lock 2020-05-21 17:28:43,279 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:43,279 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,279 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2513 runningCount=1, activeCount=1 2020-05-21 17:28:43,279 TRACE [PEWorker-15] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; cycles=0 2020-05-21 17:28:43,279 TRACE [PEWorker-15] procedure2.StateMachineProcedure: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,279 INFO [PEWorker-15] assignment.RegionStateStore: pid=2513 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=OPENING, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:43,285 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 30, totalSize: 6 bytes 2020-05-21 17:28:43,285 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 5ms 2020-05-21 17:28:43,286 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:43,287 INFO [PEWorker-15] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:43,287 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:43,287 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Stored pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE, children [pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:43,287 TRACE [PEWorker-15] wal.WALProcedureStore: Insert pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE, subproc=[pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:43,294 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:43,294 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=42.6K (347/sec) 2020-05-21 17:28:43,294 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,294 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2513 runningCount=16, activeCount=0 2020-05-21 17:28:43,294 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,294 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2515 runningCount=1, activeCount=1 2020-05-21 17:28:43,294 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,300 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:43,300 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=42.9K (350/sec) 2020-05-21 17:28:43,300 TRACE [PEWorker-15] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=false, [] 2020-05-21 17:28:43,300 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Suspend pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,300 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:43,300 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,300 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,305 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:43,306 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=43.2K (353/sec) 2020-05-21 17:28:43,306 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,311 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:43,311 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=43.6K (356/sec) 2020-05-21 17:28:43,311 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2515 runningCount=16, activeCount=0 2020-05-21 17:28:43,450 TRACE [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:43,459 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 31, totalSize: 4 bytes 2020-05-21 17:28:43,459 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 8ms 2020-05-21 17:28:43,486 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 167 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:43,486 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 167 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.14.39:44929 deadline: 1590082133486 executing as hbase 2020-05-21 17:28:43,487 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 region=rit=OPENING, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4 regionState=OPENED 2020-05-21 17:28:43,487 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,587 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=43.6K (355/sec) 2020-05-21 17:28:43,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=356 2020-05-21 17:28:43,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=43.9K (358/sec) 2020-05-21 17:28:43,594 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=true, [pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:43,594 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,594 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,594 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 167 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.14.39:44929 deadline: 1590082133486 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.14.39:44929, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:43,594 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2515, ppid=2513, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,594 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2515 runningCount=1, activeCount=1 2020-05-21 17:28:43,594 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,602 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=359 2020-05-21 17:28:43,602 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=44.3K (361/sec) 2020-05-21 17:28:43,602 INFO [PEWorker-11] assignment.RegionStateStore: pid=2513 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=OPEN, openSeqNum=261, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:43,605 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 32, totalSize: 6 bytes 2020-05-21 17:28:43,605 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:43,606 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2515, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:43,606 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Store update pid=2515, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,606 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,612 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=361 2020-05-21 17:28:43,612 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=44.6K (363/sec) 2020-05-21 17:28:43,612 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2515, ppid=2513, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:43,617 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=358 2020-05-21 17:28:43,618 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=45.0K (366/sec) 2020-05-21 17:28:43,618 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2513, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:43,618 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,623 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:43,623 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=45.4K (369/sec) 2020-05-21 17:28:43,624 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE has lock 2020-05-21 17:28:43,624 INFO [PEWorker-11] procedure2.ProcedureExecutor: Finished subprocedure pid=2515, resume processing parent pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,624 INFO [PEWorker-11] procedure2.ProcedureExecutor: Finished pid=2515, ppid=2513, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 320msec 2020-05-21 17:28:43,624 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2515 runningCount=16, activeCount=0 2020-05-21 17:28:43,624 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,624 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2513 runningCount=1, activeCount=1 2020-05-21 17:28:43,625 TRACE [PEWorker-1] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE; cycles=0 2020-05-21 17:28:43,625 TRACE [PEWorker-1] procedure2.StateMachineProcedure: pid=2513, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,625 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2513, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:43,625 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2513, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,625 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2513, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE and Delete [2515, 2514] 2020-05-21 17:28:43,630 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:43,630 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=45.8K (373/sec) 2020-05-21 17:28:43,630 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2513, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE 2020-05-21 17:28:43,635 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=419 2020-05-21 17:28:43,636 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=46.2K (376/sec) 2020-05-21 17:28:43,636 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2513, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE released the shared lock 2020-05-21 17:28:43,636 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished pid=2513, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, REOPEN/MOVE in 962msec 2020-05-21 17:28:43,636 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2513 runningCount=16, activeCount=0 2020-05-21 17:28:43,674 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=ba13fae1aa340cfe87b198638663432f, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:43,674 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, subproc=null 2020-05-21 17:28:43,775 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=46.2K (376/sec) 2020-05-21 17:28:43,781 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:43,781 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=46.6K (379/sec) 2020-05-21 17:28:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,782 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,782 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=1, activeCount=1 2020-05-21 17:28:43,783 INFO [PEWorker-5] procedure.MasterProcedureScheduler: Took xlock for pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,783 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,791 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:43,791 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=47.0K (382/sec) 2020-05-21 17:28:43,791 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:43,791 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:43,792 INFO [PEWorker-5] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:43,797 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 33, totalSize: 6 bytes 2020-05-21 17:28:43,797 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 5ms 2020-05-21 17:28:43,798 TRACE [PEWorker-5] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:43,798 INFO [PEWorker-5] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:43,798 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:43,798 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Stored pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, children [pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:43,798 TRACE [PEWorker-5] wal.WALProcedureStore: Insert pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, subproc=[pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:43,804 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:43,804 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=47.8K (388/sec) 2020-05-21 17:28:43,804 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,804 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=16, activeCount=0 2020-05-21 17:28:43,805 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,805 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2517 runningCount=1, activeCount=1 2020-05-21 17:28:43,805 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,813 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:43,813 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=48.2K (392/sec) 2020-05-21 17:28:43,814 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=false, [] 2020-05-21 17:28:43,814 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,814 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:43,814 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,814 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,819 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:43,819 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=48.6K (395/sec) 2020-05-21 17:28:43,819 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:43,828 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:43,828 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=49.0K (398/sec) 2020-05-21 17:28:43,828 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2517 runningCount=16, activeCount=0 2020-05-21 17:28:43,963 TRACE [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:43,965 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 34, totalSize: 4 bytes 2020-05-21 17:28:43,965 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:43,987 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 508 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:43,988 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 508 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082133988 executing as hbase 2020-05-21 17:28:43,988 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f regionState=CLOSED 2020-05-21 17:28:43,988 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,089 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=49.0K (397/sec) 2020-05-21 17:28:44,094 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:44,094 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=49.4K (401/sec) 2020-05-21 17:28:44,097 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=true, [pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:44,097 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,097 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:44,098 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 508 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082133988 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 110 totalTime: 110 2020-05-21 17:28:44,099 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2517, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,099 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Execute pid=2517 runningCount=1, activeCount=1 2020-05-21 17:28:44,099 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:44,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=49.8K (404/sec) 2020-05-21 17:28:44,104 INFO [PEWorker-7] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=CLOSED 2020-05-21 17:28:44,107 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 35, totalSize: 6 bytes 2020-05-21 17:28:44,107 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:44,108 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2517, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:44,108 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2517, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,108 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,113 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:44,114 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=50.3K (407/sec) 2020-05-21 17:28:44,114 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2517, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:44,120 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:44,120 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=50.7K (411/sec) 2020-05-21 17:28:44,120 TRACE [PEWorker-7] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,120 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,125 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:44,125 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=51.1K (414/sec) 2020-05-21 17:28:44,125 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE has lock 2020-05-21 17:28:44,125 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished subprocedure pid=2517, resume processing parent pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,125 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished pid=2517, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 310msec 2020-05-21 17:28:44,125 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Halt pid=2517 runningCount=16, activeCount=0 2020-05-21 17:28:44,126 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,126 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=1, activeCount=1 2020-05-21 17:28:44,126 TRACE [PEWorker-7] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:44,126 TRACE [PEWorker-7] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,126 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2516 2020-05-21 17:28:44,126 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:44,126 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,126 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,131 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:44,131 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=51.5K (417/sec) 2020-05-21 17:28:44,131 TRACE [PEWorker-7] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:44,131 TRACE [PEWorker-7] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,132 INFO [PEWorker-7] assignment.TransitRegionStateProcedure: Starting pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:44,132 TRACE [PEWorker-7] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=false, [] 2020-05-21 17:28:44,132 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Suspend pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,132 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:44,132 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,132 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,137 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:44,137 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=51.9K (420/sec) 2020-05-21 17:28:44,137 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=16, activeCount=0 2020-05-21 17:28:44,245 TRACE [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a] zookeeper.ReadOnlyZKClient: 0x1054097a to zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 inactive for 60000ms; closing (Will reconnect when new requests) 2020-05-21 17:28:44,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726599213251774 ns. 2020-05-21 17:28:44,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 116425 ns. 2020-05-21 17:28:44,282 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:44,282 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=3: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:44,282 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:44,282 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment. 2020-05-21 17:28:44,282 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=true, [pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE] 2020-05-21 17:28:44,282 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,282 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE has lock 2020-05-21 17:28:44,283 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:44,283 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,283 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=1, activeCount=1 2020-05-21 17:28:44,283 TRACE [PEWorker-2] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:44,283 TRACE [PEWorker-2] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,283 INFO [PEWorker-2] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=OPENING, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:44,286 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 36, totalSize: 6 bytes 2020-05-21 17:28:44,286 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:44,287 TRACE [PEWorker-2] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,287 INFO [PEWorker-2] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:44,287 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:44,287 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Stored pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, children [pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:44,287 TRACE [PEWorker-2] wal.WALProcedureStore: Insert pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, subproc=[pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:44,295 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:44,295 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=52.6K (426/sec) 2020-05-21 17:28:44,295 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,295 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=16, activeCount=0 2020-05-21 17:28:44,295 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,295 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2518 runningCount=1, activeCount=1 2020-05-21 17:28:44,295 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,301 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:44,301 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=52.9K (429/sec) 2020-05-21 17:28:44,301 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=false, [] 2020-05-21 17:28:44,301 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,301 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:44,301 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,301 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,306 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:44,307 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=53.3K (431/sec) 2020-05-21 17:28:44,307 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,312 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:44,312 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=53.6K (434/sec) 2020-05-21 17:28:44,312 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2518 runningCount=16, activeCount=0 2020-05-21 17:28:44,379 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore, timeout=1590082124379, delay=0 2020-05-21 17:28:44,381 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore, timeout=1590082124381, delay=0 2020-05-21 17:28:44,451 TRACE [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:44,457 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 37 exception { exception_class_name: "org.apache.hadoop.hbase.regionserver.RegionServerStoppedException" stack_trace: "org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping\n\tat org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506)\n\tat org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711)\n\tat org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)\n" do_not_retry: false }, totalSize: 830 bytes 2020-05-21 17:28:44,457 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:28:44,489 DEBUG [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 failed, try=0 org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506) at org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:99) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:89) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:364) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:341) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:351) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerStoppedException): org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506) at org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:389) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:162) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:192) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:796) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:427) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2020-05-21 17:28:44,490 WARN [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 is aborted or stopped, for safety we still need to wait until it is fully dead, try=0 2020-05-21 17:28:44,573 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082124573, delay=0 2020-05-21 17:28:44,591 TRACE [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:44,593 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 38 exception { exception_class_name: "org.apache.hadoop.hbase.regionserver.RegionServerStoppedException" stack_trace: "org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping\n\tat org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506)\n\tat org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711)\n\tat org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774)\n\tat org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413)\n\tat org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338)\n\tat org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318)\n" do_not_retry: false }, totalSize: 830 bytes 2020-05-21 17:28:44,593 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:44,595 DEBUG [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 failed, try=1 org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506) at org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:99) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:89) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:364) at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:341) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:351) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.regionserver.RegionServerStoppedException): org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1506) at org.apache.hadoop.hbase.regionserver.RSRpcServices.executeProcedures(RSRpcServices.java:3711) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$2.callBlockingMethod(AdminProtos.java:29774) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:389) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:162) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:192) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:337) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1408) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:359) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:930) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:796) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:427) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ... 1 more 2020-05-21 17:28:44,596 WARN [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 is aborted or stopped, for safety we still need to wait until it is fully dead, try=1 2020-05-21 17:28:44,624 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcServer: Disconnection /10.128.14.39:44929; # active connections=2 2020-05-21 17:28:44,644 INFO [RegionServerTracker-0] master.RegionServerTracker: RegionServer ephemeral node deleted, processing expiration [regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:44,645 DEBUG [RegionServerTracker-0] master.DeadServer: Added regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; numProcessing=1 2020-05-21 17:28:44,645 INFO [RegionServerTracker-0] master.ServerManager: Processing expiration of regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 on hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313 2020-05-21 17:28:44,647 TRACE [RegionServerTracker-0] wal.WALProcedureStore: Insert pid=2519, state=RUNNABLE:SERVER_CRASH_START; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false, subproc=null 2020-05-21 17:28:44,695 TRACE [master/hbasemaster-0:16000.Chore.1] util.MovingAverage: hbasemaster-0.hbase.hbase.svc.cluster.local,16000,1590081396313-ClusterStatusChore - start to measure at: 1726599652880478 ns. 2020-05-21 17:28:44,697 TRACE [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:44,697 DEBUG [RSProcedureDispatcher-pool3-t11] master.ServerManager: New admin connection to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:44,697 TRACE [RSProcedureDispatcher-pool3-t11] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:44,700 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: syscall:getsockopt(..) failed: Connection refused: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:44,700 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 3ms 2020-05-21 17:28:44,703 DEBUG [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 failed, try=2 org.apache.hbase.thirdparty.io.netty.channel.ExtendedClosedChannelException at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(...)(Unknown Source) 2020-05-21 17:28:44,704 WARN [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 failed due to org.apache.hbase.thirdparty.io.netty.channel.ExtendedClosedChannelException, try=2, and the server is dead, give up 2020-05-21 17:28:44,704 DEBUG [RSProcedureDispatcher-pool3-t11] assignment.RegionRemoteProcedureBase: pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure for region rit=OPENING, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, targetServer regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 is dead, SCP will interrupt us, give up 2020-05-21 17:28:44,756 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=53.6K (433/sec) 2020-05-21 17:28:44,762 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=271 2020-05-21 17:28:44,762 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=53.9K (435/sec) 2020-05-21 17:28:44,762 DEBUG [RegionServerTracker-0] procedure2.ProcedureExecutor: Stored pid=2519, state=RUNNABLE:SERVER_CRASH_START; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,763 DEBUG [RegionServerTracker-0] procedure.MasterProcedureScheduler: Add ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2519, state=RUNNABLE:SERVER_CRASH_START; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,763 INFO [RegionServerTracker-0] assignment.AssignmentManager: Scheduled SCP pid=2519 for regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 (carryingMeta=false) regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367/CRASHED/regionCount=5/lock=java.util.concurrent.locks.ReentrantReadWriteLock@61788cd8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2020-05-21 17:28:44,770 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2519, state=RUNNABLE:SERVER_CRASH_START; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,770 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2519 runningCount=1, activeCount=1 2020-05-21 17:28:44,771 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=true (2519) sharedLock=0 size=0) from run queue because: pid=2519, state=RUNNABLE:SERVER_CRASH_START; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false held exclusive lock 2020-05-21 17:28:44,771 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2519, state=RUNNABLE:SERVER_CRASH_START, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=274 2020-05-21 17:28:44,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=54.2K (437/sec) 2020-05-21 17:28:44,779 TRACE [PEWorker-3] procedure2.StateMachineProcedure: SERVER_CRASH_START pid=2519, state=RUNNABLE:SERVER_CRASH_START, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:44,779 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_START, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,779 DEBUG [PEWorker-3] master.DeadServer: Started processing regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; numProcessing=1 2020-05-21 17:28:44,781 INFO [PEWorker-3] procedure.ServerCrashProcedure: Start pid=2519, state=RUNNABLE:SERVER_CRASH_START, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,781 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2519 2020-05-21 17:28:44,781 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2519, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false as the 0th rollback step 2020-05-21 17:28:44,781 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2519, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,781 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2519, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,787 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=282 2020-05-21 17:28:44,787 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=54.4K (439/sec) 2020-05-21 17:28:44,787 TRACE [PEWorker-3] procedure2.StateMachineProcedure: SERVER_CRASH_GET_REGIONS pid=2519, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:44,787 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,788 INFO [PEWorker-3] procedure.ServerCrashProcedure: regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 had 5 regions 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure.ServerCrashProcedure: 9eda3f5973caeeba814cec96879ff797 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure.ServerCrashProcedure: ba13fae1aa340cfe87b198638663432f 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure.ServerCrashProcedure: 6d850acf1457152a77813dc3c02d3d86 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure.ServerCrashProcedure: 82ccc4215fd1fe7a2a329a0c6c297fa4 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure.ServerCrashProcedure: 8a83dfc2fd90e7fc0f6ee8c530093fa2 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2519 2020-05-21 17:28:44,788 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false as the 1th rollback step 2020-05-21 17:28:44,788 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,788 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,794 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=796 2020-05-21 17:28:44,794 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=55.2K (445/sec) 2020-05-21 17:28:44,794 TRACE [PEWorker-3] procedure2.StateMachineProcedure: SERVER_CRASH_SPLIT_LOGS pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:44,794 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,794 DEBUG [PEWorker-3] procedure.ServerCrashProcedure: Splitting WALs pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,800 DEBUG [PEWorker-3] master.MasterWalManager: Renamed region directory: hdfs://aeris/hbase/WALs/regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367-splitting 2020-05-21 17:28:44,801 INFO [PEWorker-3] master.SplitLogManager: dead splitlog workers [regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367] 2020-05-21 17:28:44,803 INFO [PEWorker-3] master.SplitLogManager: hdfs://aeris/hbase/WALs/regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367-splitting dir is empty, no logs to split. 2020-05-21 17:28:44,808 INFO [PEWorker-3] master.SplitLogManager: Finished splitting (more than or equal to) 0 (0 bytes) in 0 log files in [hdfs://aeris/hbase/WALs/regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367-splitting] in 0ms 2020-05-21 17:28:44,809 DEBUG [PEWorker-3] procedure.ServerCrashProcedure: Done splitting WALs pid=2519, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,809 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2519 2020-05-21 17:28:44,810 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false as the 2th rollback step 2020-05-21 17:28:44,810 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,810 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,817 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=800 2020-05-21 17:28:44,817 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=56.0K (452/sec) 2020-05-21 17:28:44,817 TRACE [PEWorker-3] procedure2.StateMachineProcedure: SERVER_CRASH_ASSIGN pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:44,817 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:44,818 TRACE [PEWorker-3] procedure.ServerCrashProcedure: Assigning regions [9eda3f5973caeeba814cec96879ff797, ba13fae1aa340cfe87b198638663432f, 6d850acf1457152a77813dc3c02d3d86, 82ccc4215fd1fe7a2a329a0c6c297fa4, 8a83dfc2fd90e7fc0f6ee8c530093fa2], pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:44,819 INFO [PEWorker-3] procedure.ServerCrashProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_ASSIGN, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false found RIT pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; rit=OPENING, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f 2020-05-21 17:28:44,819 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,824 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:44,824 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=56.3K (454/sec) 2020-05-21 17:28:44,825 TRACE [PEWorker-3] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=true, [pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:44,825 TRACE [PEWorker-3] procedure2.AbstractProcedureScheduler: Wake pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,825 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,825 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2518, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,825 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2518 runningCount=2, activeCount=2 2020-05-21 17:28:44,825 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,826 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH INCREMENT SET 4 java.lang.Throwable: pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,827 INFO [PEWorker-3] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN}, {pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN}, {pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN}, {pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN}] 2020-05-21 17:28:44,827 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false as the 3th rollback step 2020-05-21 17:28:44,827 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Stored pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false, children [pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN, pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN, pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN, pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN] 2020-05-21 17:28:44,827 TRACE [PEWorker-3] wal.WALProcedureStore: Insert pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false, subproc=[pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN, pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN, pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN, pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN] 2020-05-21 17:28:44,833 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=2.6K 2020-05-21 17:28:44,834 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=59.0K (475/sec) 2020-05-21 17:28:44,834 TRACE [PEWorker-10] assignment.RegionStateNode: Tracking when we are set to null rit=ABNORMALLY_CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedAbnormally(AssignmentManager.java:1726) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:306) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:58) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,834 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,835 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,835 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=2) to run queue because: the exclusive lock is not held by anyone when adding pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,835 INFO [PEWorker-10] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=ABNORMALLY_CLOSED 2020-05-21 17:28:44,835 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=2) to run queue because: the exclusive lock is not held by anyone when adding pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,835 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2519 runningCount=1, activeCount=1 2020-05-21 17:28:44,835 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2523 runningCount=3, activeCount=3 2020-05-21 17:28:44,835 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2522 runningCount=2, activeCount=2 2020-05-21 17:28:44,835 INFO [PEWorker-3] procedure.MasterProcedureScheduler: Took xlock for pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,835 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,835 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,835 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2520 runningCount=4, activeCount=4 2020-05-21 17:28:44,835 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,835 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2521 runningCount=5, activeCount=5 2020-05-21 17:28:44,836 INFO [PEWorker-12] procedure.MasterProcedureScheduler: Took xlock for pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,836 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,837 INFO [PEWorker-6] procedure.MasterProcedureScheduler: Took xlock for pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,837 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,837 INFO [PEWorker-14] procedure.MasterProcedureScheduler: Took xlock for pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,837 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,838 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 40, totalSize: 6 bytes 2020-05-21 17:28:44,838 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:44,838 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2518, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:44,838 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2518, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,838 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,845 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=1.9K 2020-05-21 17:28:44,845 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=60.9K (491/sec) 2020-05-21 17:28:44,845 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN; cycles=0 2020-05-21 17:28:44,845 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,845 TRACE [PEWorker-12] assignment.RegionStateNode: Tracking when we are set to null rit=OPEN, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.queueAssign(TransitRegionStateProcedure.java:178) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:335) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,846 INFO [PEWorker-12] assignment.TransitRegionStateProcedure: Starting pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN; rit=OPEN, location=null; forceNewPlan=true, retain=false 2020-05-21 17:28:44,846 TRACE [PEWorker-12] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=false, [] 2020-05-21 17:28:44,846 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Suspend pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,846 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN as the 4th rollback step 2020-05-21 17:28:44,846 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Store update pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,846 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,846 TRACE [PEWorker-6] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN; cycles=0 2020-05-21 17:28:44,846 TRACE [PEWorker-6] procedure2.StateMachineProcedure: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,846 TRACE [PEWorker-6] assignment.RegionStateNode: Tracking when we are set to null rit=OPEN, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.queueAssign(TransitRegionStateProcedure.java:178) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:335) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,847 INFO [PEWorker-6] assignment.TransitRegionStateProcedure: Starting pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN; rit=OPEN, location=null; forceNewPlan=true, retain=false 2020-05-21 17:28:44,847 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=false, [] 2020-05-21 17:28:44,847 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,847 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN as the 5th rollback step 2020-05-21 17:28:44,847 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,847 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,847 TRACE [PEWorker-14] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN; cycles=0 2020-05-21 17:28:44,847 TRACE [PEWorker-14] procedure2.StateMachineProcedure: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,847 TRACE [PEWorker-14] assignment.RegionStateNode: Tracking when we are set to null rit=OPEN, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.queueAssign(TransitRegionStateProcedure.java:178) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:335) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,847 INFO [PEWorker-14] assignment.TransitRegionStateProcedure: Starting pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN; rit=OPEN, location=null; forceNewPlan=true, retain=false 2020-05-21 17:28:44,847 TRACE [PEWorker-14] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 6d850acf1457152a77813dc3c02d3d86, NAME => 'aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.', STARTKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates', ENDKEY => '\x04\x08\x99\xC4\x07Devices\xC4\x0BHSH15140190\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet33\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=false, [] 2020-05-21 17:28:44,847 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Suspend pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,847 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN as the 6th rollback step 2020-05-21 17:28:44,847 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,847 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,848 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2518, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:44,849 TRACE [PEWorker-3] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN; cycles=0 2020-05-21 17:28:44,849 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,849 TRACE [PEWorker-3] assignment.RegionStateNode: Tracking when we are set to null rit=OPEN, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.queueAssign(TransitRegionStateProcedure.java:178) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:335) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,849 INFO [PEWorker-3] assignment.TransitRegionStateProcedure: Starting pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN; rit=OPEN, location=null; forceNewPlan=true, retain=false 2020-05-21 17:28:44,850 TRACE [PEWorker-3] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 8a83dfc2fd90e7fc0f6ee8c530093fa2, NAME => 'aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2.', STARTKEY => '\x01', ENDKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates'}, ready=false, [] 2020-05-21 17:28:44,850 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Suspend pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,850 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN as the 7th rollback step 2020-05-21 17:28:44,850 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,850 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,856 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=1.9K 2020-05-21 17:28:44,856 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=62.8K (506/sec) 2020-05-21 17:28:44,856 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2522 runningCount=4, activeCount=4 2020-05-21 17:28:44,856 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2520 runningCount=3, activeCount=3 2020-05-21 17:28:44,856 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2521 runningCount=2, activeCount=2 2020-05-21 17:28:44,856 TRACE [PEWorker-10] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,857 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,857 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2523 runningCount=1, activeCount=1 2020-05-21 17:28:44,881 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: shutdown connection to regionserver-0.hbase.hbase.svc.cluster.local/10.128.53.53:16020 because idle for a long time 2020-05-21 17:28:44,957 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=62.8K (506/sec) 2020-05-21 17:28:44,964 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/2, flushed=414 2020-05-21 17:28:44,964 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=63.2K (509/sec) 2020-05-21 17:28:44,964 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE has lock 2020-05-21 17:28:44,964 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished subprocedure pid=2518, resume processing parent pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,964 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished pid=2518, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 551msec 2020-05-21 17:28:44,964 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,964 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2518 runningCount=16, activeCount=0 2020-05-21 17:28:44,964 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=1, activeCount=1 2020-05-21 17:28:44,964 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:44,964 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,965 INFO [PEWorker-16] assignment.TransitRegionStateProcedure: Retry=1 of max=2147483647; pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; rit=ABNORMALLY_CLOSED, location=null 2020-05-21 17:28:44,966 TRACE [PEWorker-16] assignment.RegionStateNode: Tracking when we are set to null rit=ABNORMALLY_CLOSED, location=null, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:246) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:341) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,966 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2516 2020-05-21 17:28:44,966 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:44,966 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Store update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,966 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,972 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=418 2020-05-21 17:28:44,972 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=63.6K (512/sec) 2020-05-21 17:28:44,972 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:44,972 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,972 TRACE [PEWorker-16] assignment.RegionStateNode: Tracking when we are set to null rit=ABNORMALLY_CLOSED, location=null, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.queueAssign(TransitRegionStateProcedure.java:178) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:335) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:194) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:318) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:102) at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:962) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1662) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:44,973 INFO [PEWorker-16] assignment.TransitRegionStateProcedure: Starting pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; rit=ABNORMALLY_CLOSED, location=null; forceNewPlan=true, retain=false 2020-05-21 17:28:44,973 TRACE [PEWorker-16] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=false, [] 2020-05-21 17:28:44,973 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Suspend pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,973 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 9th rollback step 2020-05-21 17:28:44,973 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Store update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,973 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,982 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=422 2020-05-21 17:28:44,982 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=64.0K (515/sec) 2020-05-21 17:28:44,982 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=16, activeCount=0 2020-05-21 17:28:44,997 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=5 2020-05-21 17:28:44,997 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:44,997 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.', STARTKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates', ENDKEY => '\x04\x08\x99\xC4\x07Devices\xC4\x0BHSH15140190\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet33\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 6d850acf1457152a77813dc3c02d3d86, NAME => 'aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.', STARTKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates', ENDKEY => '\x04\x08\x99\xC4\x07Devices\xC4\x0BHSH15140190\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet33\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=true, [pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:44,999 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN has lock 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=true, [pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:44,999 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE has lock 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 8a83dfc2fd90e7fc0f6ee8c530093fa2, NAME => 'aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2.', STARTKEY => '\x01', ENDKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates'}, ready=true, [pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:44,999 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=2) to run queue because: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN has lock 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=true, [pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:44,999 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=2) to run queue because: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN has lock 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=true, [pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN] 2020-05-21 17:28:44,999 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:44,999 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=3) to run queue because: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN has lock 2020-05-21 17:28:45,000 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 5 -> 0msec 2020-05-21 17:28:45,000 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2523 runningCount=1, activeCount=1 2020-05-21 17:28:45,000 TRACE [PEWorker-15] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN; cycles=0 2020-05-21 17:28:45,001 TRACE [PEWorker-15] procedure2.StateMachineProcedure: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,001 INFO [PEWorker-15] assignment.RegionStateStore: pid=2523 updating hbase:meta row=8a83dfc2fd90e7fc0f6ee8c530093fa2, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,000 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2520 runningCount=2, activeCount=2 2020-05-21 17:28:45,001 TRACE [PEWorker-11] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN; cycles=0 2020-05-21 17:28:45,002 TRACE [PEWorker-11] procedure2.StateMachineProcedure: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,002 INFO [PEWorker-11] assignment.RegionStateStore: pid=2520 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,000 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,002 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2521 runningCount=3, activeCount=3 2020-05-21 17:28:45,002 TRACE [PEWorker-1] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN; cycles=0 2020-05-21 17:28:45,002 TRACE [PEWorker-1] procedure2.StateMachineProcedure: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,002 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,002 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=5, activeCount=5 2020-05-21 17:28:45,002 INFO [PEWorker-1] assignment.RegionStateStore: pid=2521 updating hbase:meta row=6d850acf1457152a77813dc3c02d3d86, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:45,002 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2522 runningCount=4, activeCount=4 2020-05-21 17:28:45,002 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN; cycles=0 2020-05-21 17:28:45,002 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,002 INFO [PEWorker-5] assignment.RegionStateStore: pid=2522 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,002 TRACE [PEWorker-4] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:45,003 TRACE [PEWorker-4] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,003 INFO [PEWorker-4] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:45,004 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 41, totalSize: 6 bytes 2020-05-21 17:28:45,004 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:45,005 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 42, totalSize: 6 bytes 2020-05-21 17:28:45,005 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:45,005 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,005 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,005 INFO [PEWorker-15] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:45,005 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2523, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN as the 8th rollback step 2020-05-21 17:28:45,005 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Stored pid=2523, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN, children [pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,005 TRACE [PEWorker-15] wal.WALProcedureStore: Insert pid=2523, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN, subproc=[pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,005 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 43, totalSize: 6 bytes 2020-05-21 17:28:45,005 INFO [PEWorker-11] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:45,006 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 4ms 2020-05-21 17:28:45,006 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2520, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN as the 9th rollback step 2020-05-21 17:28:45,006 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Stored pid=2520, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN, children [pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,006 TRACE [PEWorker-11] wal.WALProcedureStore: Insert pid=2520, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN, subproc=[pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,006 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 44, totalSize: 6 bytes 2020-05-21 17:28:45,006 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:45,006 TRACE [PEWorker-5] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,006 TRACE [PEWorker-1] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,006 INFO [PEWorker-1] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:45,007 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2521, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN as the 10th rollback step 2020-05-21 17:28:45,007 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Stored pid=2521, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN, children [pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,007 TRACE [PEWorker-1] wal.WALProcedureStore: Insert pid=2521, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN, subproc=[pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,006 INFO [PEWorker-5] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:45,007 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2522, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN as the 11th rollback step 2020-05-21 17:28:45,007 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Stored pid=2522, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN, children [pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,007 TRACE [PEWorker-5] wal.WALProcedureStore: Insert pid=2522, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN, subproc=[pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,006 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 45, totalSize: 6 bytes 2020-05-21 17:28:45,007 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 4ms 2020-05-21 17:28:45,007 TRACE [PEWorker-4] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,008 INFO [PEWorker-4] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:45,008 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 10th rollback step 2020-05-21 17:28:45,008 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Stored pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, children [pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,008 TRACE [PEWorker-4] wal.WALProcedureStore: Insert pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE, subproc=[pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,013 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=3.9K 2020-05-21 17:28:45,013 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=67.9K (547/sec) 2020-05-21 17:28:45,014 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2523 runningCount=4, activeCount=4 2020-05-21 17:28:45,014 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2525 runningCount=5, activeCount=5 2020-05-21 17:28:45,014 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=2) to run queue because: the exclusive lock is not held by anyone when adding pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2520 runningCount=3, activeCount=3 2020-05-21 17:28:45,014 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2524 runningCount=4, activeCount=4 2020-05-21 17:28:45,014 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=2) to run queue because: the exclusive lock is not held by anyone when adding pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,014 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2522 runningCount=4, activeCount=4 2020-05-21 17:28:45,015 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2528 runningCount=5, activeCount=5 2020-05-21 17:28:45,015 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=4, activeCount=4 2020-05-21 17:28:45,015 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2526 runningCount=5, activeCount=5 2020-05-21 17:28:45,015 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2521 runningCount=4, activeCount=4 2020-05-21 17:28:45,015 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,015 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2527 runningCount=5, activeCount=5 2020-05-21 17:28:45,015 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,022 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=4/4, flushed=1.4K 2020-05-21 17:28:45,022 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=69.3K (558/sec) 2020-05-21 17:28:45,023 TRACE [PEWorker-15] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 8a83dfc2fd90e7fc0f6ee8c530093fa2, NAME => 'aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2.', STARTKEY => '\x01', ENDKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates'}, ready=false, [] 2020-05-21 17:28:45,023 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Suspend pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,023 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 12th rollback step 2020-05-21 17:28:45,023 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,023 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,023 TRACE [PEWorker-11] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=false, [] 2020-05-21 17:28:45,023 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Suspend pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,023 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 13th rollback step 2020-05-21 17:28:45,023 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Store update pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,023 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=false, [] 2020-05-21 17:28:45,024 TRACE [PEWorker-5] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=false, [] 2020-05-21 17:28:45,024 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Suspend pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 11th rollback step 2020-05-21 17:28:45,024 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 14th rollback step 2020-05-21 17:28:45,024 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,024 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,060 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 509 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3640 bytes 2020-05-21 17:28:45,061 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 509 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.9.37:50537 deadline: 1590082135061 executing as hbase 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144617 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:45,062 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:45,063 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 509 service: RegionServerStatusService methodName: RegionServerReport size: 3.6 K connection: 10.128.9.37:50537 deadline: 1590082135061 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 447 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:45,124 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=4 , totalSynced=69.3K (558/sec) 2020-05-21 17:28:45,130 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=4/5, flushed=1.6K 2020-05-21 17:28:45,131 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=70.9K (570/sec) 2020-05-21 17:28:45,131 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,131 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,131 TRACE [PEWorker-1] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 6d850acf1457152a77813dc3c02d3d86, NAME => 'aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.', STARTKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates', ENDKEY => '\x04\x08\x99\xC4\x07Devices\xC4\x0BHSH15140190\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet33\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=false, [] 2020-05-21 17:28:45,131 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Suspend pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,131 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 15th rollback step 2020-05-21 17:28:45,131 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,132 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,132 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,140 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=1.9K 2020-05-21 17:28:45,140 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=72.8K (586/sec) 2020-05-21 17:28:45,140 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2525 runningCount=4, activeCount=4 2020-05-21 17:28:45,140 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2524 runningCount=3, activeCount=3 2020-05-21 17:28:45,140 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,140 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,141 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2528 runningCount=2, activeCount=2 2020-05-21 17:28:45,146 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=856 2020-05-21 17:28:45,146 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=73.7K (593/sec) 2020-05-21 17:28:45,146 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2527 runningCount=1, activeCount=1 2020-05-21 17:28:45,146 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2526 runningCount=16, activeCount=0 2020-05-21 17:28:45,152 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 507 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3920 bytes 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 507 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.53.53:44503 deadline: 1590082135152 executing as hbase 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:45,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: bf4b11823c10883a42ef94., aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:45,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 507 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.53.53:44503 deadline: 1590082135152 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 528 } connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:28:45,173 TRACE [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: Building request with operations count=3 2020-05-21 17:28:45,174 TRACE [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: Building request with operations count=2 2020-05-21 17:28:45,185 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 47, totalSize: 4 bytes 2020-05-21 17:28:45,186 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 11ms 2020-05-21 17:28:45,191 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 46, totalSize: 4 bytes 2020-05-21 17:28:45,191 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 17ms 2020-05-21 17:28:45,203 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 510 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:45,203 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 508 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:45,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 510 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082135203 executing as hbase 2020-05-21 17:28:45,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 508 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.53.53:44503 deadline: 1590082135203 executing as hbase 2020-05-21 17:28:45,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f regionState=OPENED 2020-05-21 17:28:45,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,205 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4 regionState=OPENED 2020-05-21 17:28:45,205 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,232 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 511 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:45,244 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 509 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 325 bytes 2020-05-21 17:28:45,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726600213250766 ns. 2020-05-21 17:28:45,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 139790 ns. 2020-05-21 17:28:45,277 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 512 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 232 bytes 2020-05-21 17:28:45,305 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=2 , totalSynced=73.7K (592/sec) 2020-05-21 17:28:45,310 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/16, flushed=712 2020-05-21 17:28:45,311 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=74.4K (597/sec) 2020-05-21 17:28:45,311 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => ba13fae1aa340cfe87b198638663432f, NAME => 'aeris_test_backup_restore,\x02,1581768003353.ba13fae1aa340cfe87b198638663432f.', STARTKEY => '\x02', ENDKEY => '\x03'}, ready=true, [pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,311 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,311 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,311 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2528, ppid=2516, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,311 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2528 runningCount=1, activeCount=1 2020-05-21 17:28:45,311 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 508 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.53.53:44503 deadline: 1590082135203 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 107 totalTime: 108 2020-05-21 17:28:45,311 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,311 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 511 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082135232 executing as hbase 2020-05-21 17:28:45,312 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797 regionState=OPENED 2020-05-21 17:28:45,312 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,312 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 82ccc4215fd1fe7a2a329a0c6c297fa4, NAME => 'aeris_test_backup_restore,\x01,1581768003353.82ccc4215fd1fe7a2a329a0c6c297fa4.', STARTKEY => '\x01', ENDKEY => '\x02'}, ready=true, [pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,312 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,312 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,312 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2526, ppid=2522, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,312 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 510 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082135203 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 1 processingTime: 108 totalTime: 109 2020-05-21 17:28:45,312 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2526 runningCount=2, activeCount=2 2020-05-21 17:28:45,312 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,313 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 509 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 325 connection: 10.128.53.53:44503 deadline: 1590082135244 executing as hbase 2020-05-21 17:28:45,313 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86 regionState=OPENED 2020-05-21 17:28:45,313 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,318 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/1, flushed=715 2020-05-21 17:28:45,318 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=75.1K (603/sec) 2020-05-21 17:28:45,318 INFO [PEWorker-12] assignment.RegionStateStore: pid=2516 updating hbase:meta row=ba13fae1aa340cfe87b198638663432f, regionState=OPEN, openSeqNum=252, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:45,319 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 9eda3f5973caeeba814cec96879ff797, NAME => 'aeris_test_backup_restore,\x09,1581768003353.9eda3f5973caeeba814cec96879ff797.', STARTKEY => '\x09', ENDKEY => '\x0A'}, ready=true, [pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,319 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,319 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,319 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2524, ppid=2520, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,319 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2524 runningCount=3, activeCount=3 2020-05-21 17:28:45,319 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,320 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 511 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082135232 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 79 processingTime: 8 totalTime: 87 2020-05-21 17:28:45,320 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 512 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 232 connection: 10.128.9.37:50537 deadline: 1590082135277 executing as hbase 2020-05-21 17:28:45,320 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2 regionState=OPENED 2020-05-21 17:28:45,320 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,323 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 48, totalSize: 6 bytes 2020-05-21 17:28:45,323 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 4ms 2020-05-21 17:28:45,323 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2528, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 12th rollback step 2020-05-21 17:28:45,323 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Store update pid=2528, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,323 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,325 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=871 2020-05-21 17:28:45,325 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=75.9K (610/sec) 2020-05-21 17:28:45,326 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 6d850acf1457152a77813dc3c02d3d86, NAME => 'aeris_v2,\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates,1587075428855.6d850acf1457152a77813dc3c02d3d86.', STARTKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates', ENDKEY => '\x04\x08\x99\xC4\x07Devices\xC4\x0BHSH15140190\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet33\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=true, [pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,326 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,326 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,326 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 509 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 325 connection: 10.128.53.53:44503 deadline: 1590082135244 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 69 processingTime: 13 totalTime: 82 2020-05-21 17:28:45,326 INFO [PEWorker-14] assignment.RegionStateStore: pid=2520 updating hbase:meta row=9eda3f5973caeeba814cec96879ff797, regionState=OPEN, openSeqNum=279, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,326 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2527, ppid=2521, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,327 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2527 runningCount=4, activeCount=4 2020-05-21 17:28:45,327 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,329 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 49, totalSize: 6 bytes 2020-05-21 17:28:45,329 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:45,329 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2524, ppid=2520, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 16th rollback step 2020-05-21 17:28:45,330 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2524, ppid=2520, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,330 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,334 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=3/3, flushed=1.3K 2020-05-21 17:28:45,334 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=77.2K (620/sec) 2020-05-21 17:28:45,334 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2528, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,334 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 8a83dfc2fd90e7fc0f6ee8c530093fa2, NAME => 'aeris_v2,\x01,1587075427456.8a83dfc2fd90e7fc0f6ee8c530093fa2.', STARTKEY => '\x01', ENDKEY => '\x03\x08\x97\xC4\x07Devices\xC4\x0BSSJ18432083\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet28\xC4\x05rates'}, ready=true, [pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:45,334 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,334 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,334 INFO [PEWorker-3] assignment.RegionStateStore: pid=2521 updating hbase:meta row=6d850acf1457152a77813dc3c02d3d86, regionState=OPEN, openSeqNum=116813584, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:45,336 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2525, ppid=2523, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,336 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2525 runningCount=5, activeCount=5 2020-05-21 17:28:45,336 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,336 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 512 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 232 connection: 10.128.9.37:50537 deadline: 1590082135277 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 43 processingTime: 16 totalTime: 59 2020-05-21 17:28:45,338 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 50, totalSize: 6 bytes 2020-05-21 17:28:45,338 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:45,338 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2527, ppid=2521, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 17th rollback step 2020-05-21 17:28:45,338 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2527, ppid=2521, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,338 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,342 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=4/4, flushed=1.5K 2020-05-21 17:28:45,342 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=78.7K (632/sec) 2020-05-21 17:28:45,342 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2524, ppid=2520, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,343 INFO [PEWorker-6] assignment.RegionStateStore: pid=2522 updating hbase:meta row=82ccc4215fd1fe7a2a329a0c6c297fa4, regionState=OPEN, openSeqNum=264, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,343 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2516, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,344 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,344 INFO [PEWorker-10] assignment.RegionStateStore: pid=2523 updating hbase:meta row=8a83dfc2fd90e7fc0f6ee8c530093fa2, regionState=OPEN, openSeqNum=142840889, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,347 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 52, totalSize: 6 bytes 2020-05-21 17:28:45,347 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:45,347 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2525, ppid=2523, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 18th rollback step 2020-05-21 17:28:45,347 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2525, ppid=2523, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,347 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,348 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 51, totalSize: 6 bytes 2020-05-21 17:28:45,348 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 5ms 2020-05-21 17:28:45,348 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2526, ppid=2522, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 19th rollback step 2020-05-21 17:28:45,348 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2526, ppid=2522, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,348 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,355 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=2.0K 2020-05-21 17:28:45,356 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=80.7K (648/sec) 2020-05-21 17:28:45,356 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2527, ppid=2521, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,356 TRACE [PEWorker-14] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2520, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,356 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,356 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE has lock 2020-05-21 17:28:45,357 INFO [PEWorker-12] procedure2.ProcedureExecutor: Finished subprocedure pid=2528, resume processing parent pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,357 INFO [PEWorker-12] procedure2.ProcedureExecutor: Finished pid=2528, ppid=2516, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 316msec 2020-05-21 17:28:45,357 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2528 runningCount=4, activeCount=4 2020-05-21 17:28:45,357 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,357 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2525, ppid=2523, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,357 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2516 runningCount=5, activeCount=5 2020-05-21 17:28:45,357 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:45,357 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2516, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,357 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2516, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE as the 13th rollback step 2020-05-21 17:28:45,357 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2526, ppid=2522, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:45,357 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Store update pid=2516, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,357 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2516, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE and Delete [2517, 2528, 2518] 2020-05-21 17:28:45,365 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=2.0K 2020-05-21 17:28:45,365 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=82.7K (664/sec) 2020-05-21 17:28:45,365 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2521, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,366 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,366 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=1) to run queue because: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN has lock 2020-05-21 17:28:45,366 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished subprocedure pid=2524, resume processing parent pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,366 TRACE [PEWorker-10] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2523, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,366 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2516, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE 2020-05-21 17:28:45,366 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=3 size=0) from run queue because: queue is empty after polling out pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,367 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2520 runningCount=6, activeCount=6 2020-05-21 17:28:45,367 TRACE [PEWorker-9] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN; cycles=0 2020-05-21 17:28:45,367 TRACE [PEWorker-9] procedure2.StateMachineProcedure: pid=2520, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,367 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2520, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN as the 20th rollback step 2020-05-21 17:28:45,367 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2520, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,367 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2520, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,366 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2524, ppid=2520, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 324msec 2020-05-21 17:28:45,366 TRACE [PEWorker-6] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2522, ppid=2519, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,367 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,367 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2524 runningCount=5, activeCount=5 2020-05-21 17:28:45,366 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,375 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=5/5, flushed=2.0K 2020-05-21 17:28:45,375 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=84.7K (680/sec) 2020-05-21 17:28:45,375 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN has lock 2020-05-21 17:28:45,375 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished subprocedure pid=2527, resume processing parent pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,375 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished pid=2527, ppid=2521, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 332msec 2020-05-21 17:28:45,375 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2527 runningCount=4, activeCount=4 2020-05-21 17:28:45,375 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,375 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2521 runningCount=5, activeCount=5 2020-05-21 17:28:45,375 TRACE [PEWorker-3] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN; cycles=0 2020-05-21 17:28:45,375 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2521, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,375 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2521, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN as the 21th rollback step 2020-05-21 17:28:45,375 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2521, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,375 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2521, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,376 INFO [PEWorker-16] procedure2.ProcedureExecutor: Finished pid=2516, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=ba13fae1aa340cfe87b198638663432f, REOPEN/MOVE in 1.6830sec 2020-05-21 17:28:45,376 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2516 runningCount=4, activeCount=4 2020-05-21 17:28:45,376 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2520, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN 2020-05-21 17:28:45,376 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=2 size=1) to run queue because: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN has lock 2020-05-21 17:28:45,376 INFO [PEWorker-6] procedure2.ProcedureExecutor: Finished subprocedure pid=2526, resume processing parent pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,376 INFO [PEWorker-6] procedure2.ProcedureExecutor: Finished pid=2526, ppid=2522, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 342msec 2020-05-21 17:28:45,376 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2526 runningCount=3, activeCount=3 2020-05-21 17:28:45,376 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,376 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2522 runningCount=4, activeCount=4 2020-05-21 17:28:45,376 TRACE [PEWorker-6] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN; cycles=0 2020-05-21 17:28:45,376 TRACE [PEWorker-6] procedure2.StateMachineProcedure: pid=2522, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,376 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2522, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN as the 22th rollback step 2020-05-21 17:28:45,376 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2522, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,376 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2522, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,377 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=2 size=1) to run queue because: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN has lock 2020-05-21 17:28:45,377 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished subprocedure pid=2525, resume processing parent pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,377 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished pid=2525, ppid=2523, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 342msec 2020-05-21 17:28:45,377 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2525 runningCount=3, activeCount=3 2020-05-21 17:28:45,377 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=2 size=0) from run queue because: queue is empty after polling out pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,377 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2523 runningCount=4, activeCount=4 2020-05-21 17:28:45,377 TRACE [PEWorker-8] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN; cycles=0 2020-05-21 17:28:45,377 TRACE [PEWorker-8] procedure2.StateMachineProcedure: pid=2523, ppid=2519, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,377 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2523, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN as the 23th rollback step 2020-05-21 17:28:45,377 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2523, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,377 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2523, ppid=2519, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=4/4, flushed=1.6K 2020-05-21 17:28:45,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=86.3K (693/sec) 2020-05-21 17:28:45,382 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2521, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN 2020-05-21 17:28:45,383 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2522, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN 2020-05-21 17:28:45,383 TRACE [PEWorker-9] procedure2.Procedure: CHILD LATCH DECREMENT 3 java.lang.Throwable: pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,383 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2523, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN 2020-05-21 17:28:45,383 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2520, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=9eda3f5973caeeba814cec96879ff797, ASSIGN in 542msec 2020-05-21 17:28:45,383 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2520 runningCount=3, activeCount=3 2020-05-21 17:28:45,385 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=cb592da589b3dd3af1fd129c458e2404, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:45,385 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE, subproc=null 2020-05-21 17:28:45,389 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=3/3, flushed=1.3K 2020-05-21 17:28:45,390 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=87.6K (703/sec) 2020-05-21 17:28:45,390 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH DECREMENT 2 java.lang.Throwable: pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,392 INFO [PEWorker-3] procedure2.ProcedureExecutor: Finished pid=2521, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=6d850acf1457152a77813dc3c02d3d86, ASSIGN in 550msec 2020-05-21 17:28:45,392 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2521 runningCount=2, activeCount=2 2020-05-21 17:28:45,390 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2522, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN released the shared lock 2020-05-21 17:28:45,392 TRACE [PEWorker-6] procedure2.Procedure: CHILD LATCH DECREMENT 1 java.lang.Throwable: pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,392 INFO [PEWorker-6] procedure2.ProcedureExecutor: Finished pid=2522, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=82ccc4215fd1fe7a2a329a0c6c297fa4, ASSIGN in 551msec 2020-05-21 17:28:45,392 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2522 runningCount=1, activeCount=1 2020-05-21 17:28:45,392 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2523, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN released the shared lock 2020-05-21 17:28:45,392 TRACE [PEWorker-8] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2519, state=WAITING:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,392 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,493 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=2 , totalSynced=87.6K (703/sec) 2020-05-21 17:28:45,502 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/1, flushed=1.3K 2020-05-21 17:28:45,502 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=88.9K (713/sec) 2020-05-21 17:28:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,503 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Add ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=true (2519) sharedLock=0 size=1) to run queue because: pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false has lock 2020-05-21 17:28:45,503 INFO [PEWorker-8] procedure2.ProcedureExecutor: Finished subprocedure pid=2523, resume processing parent pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,503 INFO [PEWorker-8] procedure2.ProcedureExecutor: Finished pid=2523, ppid=2519, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=8a83dfc2fd90e7fc0f6ee8c530093fa2, ASSIGN in 551msec 2020-05-21 17:28:45,503 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2523 runningCount=16, activeCount=0 2020-05-21 17:28:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,503 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=true (2519) sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,503 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2519 runningCount=1, activeCount=1 2020-05-21 17:28:45,503 TRACE [PEWorker-13] procedure2.StateMachineProcedure: SERVER_CRASH_FINISH pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false; cycles=0 2020-05-21 17:28:45,503 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2519, state=RUNNABLE:SERVER_CRASH_FINISH, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,503 INFO [PEWorker-13] procedure.ServerCrashProcedure: removed crashed server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 after splitting done 2020-05-21 17:28:45,505 DEBUG [PEWorker-13] master.DeadServer: Finished processing regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; numProcessing=0 2020-05-21 17:28:45,505 DEBUG [PEWorker-13] master.DeadServer: Removed regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 ; numProcessing=0 2020-05-21 17:28:45,505 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2519, state=SUCCESS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false as the 24th rollback step 2020-05-21 17:28:45,505 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2519, state=SUCCESS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,505 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2519, state=SUCCESS, locked=true; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false and Delete [2522, 2524, 2526, 2525, 2527, 2520, 2521, 2523] 2020-05-21 17:28:45,505 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,505 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2529 runningCount=2, activeCount=2 2020-05-21 17:28:45,506 INFO [PEWorker-8] procedure.MasterProcedureScheduler: Took xlock for pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,506 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,512 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=1.4K 2020-05-21 17:28:45,512 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=90.3K (724/sec) 2020-05-21 17:28:45,512 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2519, state=SUCCESS; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:28:45,512 TRACE [PEWorker-8] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; cycles=0 2020-05-21 17:28:45,512 TRACE [PEWorker-8] procedure2.StateMachineProcedure: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:45,512 INFO [PEWorker-8] assignment.RegionStateStore: pid=2529 updating hbase:meta row=cb592da589b3dd3af1fd129c458e2404, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:45,515 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 53, totalSize: 6 bytes 2020-05-21 17:28:45,515 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:45,515 TRACE [PEWorker-8] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:45,515 INFO [PEWorker-8] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:45,516 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:45,516 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Stored pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE, children [pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:45,516 TRACE [PEWorker-8] wal.WALProcedureStore: Insert pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE, subproc=[pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:45,523 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=1.9K 2020-05-21 17:28:45,524 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=92.2K (739/sec) 2020-05-21 17:28:45,524 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=false sharedLock=0 size=0) to run queue because: pid=2519, state=SUCCESS; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false released exclusive lock 2020-05-21 17:28:45,524 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2519, state=SUCCESS; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false in 858msec 2020-05-21 17:28:45,524 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,525 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2529 runningCount=1, activeCount=1 2020-05-21 17:28:45,525 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove ServerQueue(regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, xlock=true (2519) sharedLock=0 size=0) from run queue because: clean up server queue after pid=2519, state=SUCCESS; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false completed 2020-05-21 17:28:45,525 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,525 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2530 runningCount=2, activeCount=2 2020-05-21 17:28:45,525 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2519 runningCount=1, activeCount=1 2020-05-21 17:28:45,526 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,531 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=568 2020-05-21 17:28:45,531 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=92.7K (743/sec) 2020-05-21 17:28:45,532 TRACE [PEWorker-11] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=false, [] 2020-05-21 17:28:45,532 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Suspend pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,532 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:45,532 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Store update pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,532 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,537 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=570 2020-05-21 17:28:45,538 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=93.3K (748/sec) 2020-05-21 17:28:45,538 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,545 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=567 2020-05-21 17:28:45,545 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=93.8K (752/sec) 2020-05-21 17:28:45,545 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2530 runningCount=16, activeCount=0 2020-05-21 17:28:45,682 TRACE [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:45,684 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 54, totalSize: 4 bytes 2020-05-21 17:28:45,685 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 3ms 2020-05-21 17:28:45,875 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 513 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 320 bytes 2020-05-21 17:28:45,875 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 513 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 320 connection: 10.128.9.37:50537 deadline: 1590082135875 executing as hbase 2020-05-21 17:28:45,877 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404 regionState=CLOSED 2020-05-21 17:28:45,877 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,979 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=93.8K (750/sec) 2020-05-21 17:28:45,985 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=580 2020-05-21 17:28:45,985 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=94.4K (754/sec) 2020-05-21 17:28:45,986 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=true, [pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:45,986 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,986 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,986 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:45,986 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2530, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:45,999 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2530 runningCount=1, activeCount=1 2020-05-21 17:28:45,999 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,000 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 513 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 320 connection: 10.128.9.37:50537 deadline: 1590082135875 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 111 totalTime: 111 2020-05-21 17:28:46,007 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=583 2020-05-21 17:28:46,008 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=95.0K (759/sec) 2020-05-21 17:28:46,008 INFO [PEWorker-5] assignment.RegionStateStore: pid=2529 updating hbase:meta row=cb592da589b3dd3af1fd129c458e2404, regionState=CLOSED 2020-05-21 17:28:46,011 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 55, totalSize: 6 bytes 2020-05-21 17:28:46,011 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:46,011 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2530, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:46,011 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2530, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,011 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,016 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=585 2020-05-21 17:28:46,017 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=95.5K (763/sec) 2020-05-21 17:28:46,017 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2530, ppid=2529, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,023 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=582 2020-05-21 17:28:46,024 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=96.1K (768/sec) 2020-05-21 17:28:46,024 TRACE [PEWorker-5] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:46,024 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,030 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=556 2020-05-21 17:28:46,030 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=96.6K (772/sec) 2020-05-21 17:28:46,030 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE has lock 2020-05-21 17:28:46,031 INFO [PEWorker-5] procedure2.ProcedureExecutor: Finished subprocedure pid=2530, resume processing parent pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,031 INFO [PEWorker-5] procedure2.ProcedureExecutor: Finished pid=2530, ppid=2529, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 496msec 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2530 runningCount=16, activeCount=0 2020-05-21 17:28:46,031 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2529 runningCount=1, activeCount=1 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; cycles=0 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2529 2020-05-21 17:28:46,031 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:46,031 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,031 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,038 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=560 2020-05-21 17:28:46,038 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=97.2K (776/sec) 2020-05-21 17:28:46,038 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; cycles=0 2020-05-21 17:28:46,038 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,038 INFO [PEWorker-5] assignment.TransitRegionStateProcedure: Starting pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:46,038 TRACE [PEWorker-5] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=false, [] 2020-05-21 17:28:46,038 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Suspend pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,038 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:46,038 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,038 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,046 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=564 2020-05-21 17:28:46,046 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=97.7K (781/sec) 2020-05-21 17:28:46,046 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2529 runningCount=16, activeCount=0 2020-05-21 17:28:46,188 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:46,188 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:46,189 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:46,194 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:46,194 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=true, [pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE] 2020-05-21 17:28:46,194 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,194 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE has lock 2020-05-21 17:28:46,194 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:46,201 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,201 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2529 runningCount=1, activeCount=1 2020-05-21 17:28:46,201 TRACE [PEWorker-4] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; cycles=0 2020-05-21 17:28:46,201 TRACE [PEWorker-4] procedure2.StateMachineProcedure: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,201 INFO [PEWorker-4] assignment.RegionStateStore: pid=2529 updating hbase:meta row=cb592da589b3dd3af1fd129c458e2404, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:46,209 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 56, totalSize: 6 bytes 2020-05-21 17:28:46,209 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 7ms 2020-05-21 17:28:46,209 TRACE [PEWorker-4] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:46,209 INFO [PEWorker-4] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:46,210 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:46,210 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Stored pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE, children [pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:46,210 TRACE [PEWorker-4] wal.WALProcedureStore: Insert pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE, subproc=[pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:46,215 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=1.0K 2020-05-21 17:28:46,216 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=98.8K (788/sec) 2020-05-21 17:28:46,216 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,216 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2529 runningCount=16, activeCount=0 2020-05-21 17:28:46,216 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,216 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2531 runningCount=1, activeCount=1 2020-05-21 17:28:46,216 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,222 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=506 2020-05-21 17:28:46,222 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=99.3K (792/sec) 2020-05-21 17:28:46,225 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=false, [] 2020-05-21 17:28:46,225 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,225 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:46,225 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,225 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,230 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=508 2020-05-21 17:28:46,230 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=99.8K (796/sec) 2020-05-21 17:28:46,231 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,237 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=505 2020-05-21 17:28:46,237 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=100.3K (799/sec) 2020-05-21 17:28:46,237 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2531 runningCount=16, activeCount=0 2020-05-21 17:28:46,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726601213216997 ns. 2020-05-21 17:28:46,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 82893 ns. 2020-05-21 17:28:46,375 TRACE [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:46,382 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 57, totalSize: 4 bytes 2020-05-21 17:28:46,382 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 7ms 2020-05-21 17:28:46,428 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 510 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 325 bytes 2020-05-21 17:28:46,429 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 510 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 325 connection: 10.128.53.53:44503 deadline: 1590082136428 executing as hbase 2020-05-21 17:28:46,430 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404 regionState=OPENED 2020-05-21 17:28:46,430 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,531 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=100.3K (798/sec) 2020-05-21 17:28:46,537 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=512 2020-05-21 17:28:46,537 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=100.8K (802/sec) 2020-05-21 17:28:46,537 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => cb592da589b3dd3af1fd129c458e2404, NAME => 'aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404.', STARTKEY => '\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates', ENDKEY => '\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m'}, ready=true, [pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:46,537 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,537 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,537 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2531, ppid=2529, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,537 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 510 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 325 connection: 10.128.53.53:44503 deadline: 1590082136428 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 108 totalTime: 109 2020-05-21 17:28:46,538 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2531 runningCount=1, activeCount=1 2020-05-21 17:28:46,538 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,543 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=515 2020-05-21 17:28:46,544 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=101.3K (806/sec) 2020-05-21 17:28:46,544 INFO [PEWorker-14] assignment.RegionStateStore: pid=2529 updating hbase:meta row=cb592da589b3dd3af1fd129c458e2404, regionState=OPEN, openSeqNum=80144965, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:46,547 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 58, totalSize: 6 bytes 2020-05-21 17:28:46,547 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:46,547 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2531, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:46,547 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2531, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,547 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=517 2020-05-21 17:28:46,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=101.8K (810/sec) 2020-05-21 17:28:46,553 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2531, ppid=2529, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:46,558 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=514 2020-05-21 17:28:46,558 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=102.3K (813/sec) 2020-05-21 17:28:46,558 TRACE [PEWorker-14] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2529, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:46,558 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,564 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=568 2020-05-21 17:28:46,565 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=102.8K (818/sec) 2020-05-21 17:28:46,565 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE has lock 2020-05-21 17:28:46,565 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,565 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Execute pid=2529 runningCount=2, activeCount=2 2020-05-21 17:28:46,565 TRACE [PEWorker-7] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE; cycles=0 2020-05-21 17:28:46,565 TRACE [PEWorker-7] procedure2.StateMachineProcedure: pid=2529, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,565 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2529, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:46,565 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2529, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,565 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2529, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE and Delete [2530, 2531] 2020-05-21 17:28:46,566 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished subprocedure pid=2531, resume processing parent pid=2529, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,566 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2531, ppid=2529, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 338msec 2020-05-21 17:28:46,566 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2531 runningCount=1, activeCount=1 2020-05-21 17:28:46,604 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=20404052b3c0674a71c9f9da299b9e17, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:46,605 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE, subproc=null 2020-05-21 17:28:46,605 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 39msec, slotIndex=2 , totalSynced=102.8K (818/sec) 2020-05-21 17:28:46,611 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=975 2020-05-21 17:28:46,611 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=103.8K (825/sec) 2020-05-21 17:28:46,611 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2529, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE 2020-05-21 17:28:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,613 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,613 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2532 runningCount=2, activeCount=2 2020-05-21 17:28:46,613 INFO [PEWorker-16] procedure.MasterProcedureScheduler: Took xlock for pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,613 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,619 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=573 2020-05-21 17:28:46,620 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=104.3K (830/sec) 2020-05-21 17:28:46,620 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2529, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE released the shared lock 2020-05-21 17:28:46,620 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished pid=2529, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=cb592da589b3dd3af1fd129c458e2404, REOPEN/MOVE in 1.1800sec 2020-05-21 17:28:46,620 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Halt pid=2529 runningCount=1, activeCount=1 2020-05-21 17:28:46,720 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=104.3K (829/sec) 2020-05-21 17:28:46,725 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/2, flushed=393 2020-05-21 17:28:46,726 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=104.7K (832/sec) 2020-05-21 17:28:46,726 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; cycles=0 2020-05-21 17:28:46,726 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:46,727 INFO [PEWorker-16] assignment.RegionStateStore: pid=2532 updating hbase:meta row=20404052b3c0674a71c9f9da299b9e17, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:46,731 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 59, totalSize: 6 bytes 2020-05-21 17:28:46,731 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 4ms 2020-05-21 17:28:46,732 TRACE [PEWorker-16] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:46,732 INFO [PEWorker-16] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:46,732 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:46,732 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Stored pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE, children [pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:46,732 TRACE [PEWorker-16] wal.WALProcedureStore: Insert pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE, subproc=[pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:46,742 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=807 2020-05-21 17:28:46,742 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=105.5K (838/sec) 2020-05-21 17:28:46,742 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,742 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2532 runningCount=16, activeCount=0 2020-05-21 17:28:46,742 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,742 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2533 runningCount=1, activeCount=1 2020-05-21 17:28:46,743 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,750 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:46,750 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=105.9K (841/sec) 2020-05-21 17:28:46,750 TRACE [PEWorker-2] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=false, [] 2020-05-21 17:28:46,750 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Suspend pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,750 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:46,750 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Store update pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,750 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,758 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=415 2020-05-21 17:28:46,758 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=106.3K (844/sec) 2020-05-21 17:28:46,758 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:46,763 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=412 2020-05-21 17:28:46,763 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=106.7K (848/sec) 2020-05-21 17:28:46,763 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2533 runningCount=16, activeCount=0 2020-05-21 17:28:46,901 TRACE [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:46,904 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 60, totalSize: 4 bytes 2020-05-21 17:28:46,904 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 3ms 2020-05-21 17:28:46,920 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 514 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 164 bytes 2020-05-21 17:28:46,920 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 514 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 164 connection: 10.128.9.37:50537 deadline: 1590082136920 executing as hbase 2020-05-21 17:28:46,921 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17 regionState=CLOSED 2020-05-21 17:28:46,921 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,022 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=106.7K (846/sec) 2020-05-21 17:28:47,027 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=426 2020-05-21 17:28:47,027 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=107.1K (849/sec) 2020-05-21 17:28:47,028 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=true, [pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:47,028 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,028 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,028 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:47,028 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 514 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 164 connection: 10.128.9.37:50537 deadline: 1590082136920 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:47,029 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2533, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,029 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2533 runningCount=1, activeCount=1 2020-05-21 17:28:47,029 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,035 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:47,035 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=107.6K (852/sec) 2020-05-21 17:28:47,036 INFO [PEWorker-10] assignment.RegionStateStore: pid=2532 updating hbase:meta row=20404052b3c0674a71c9f9da299b9e17, regionState=CLOSED 2020-05-21 17:28:47,038 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 61, totalSize: 6 bytes 2020-05-21 17:28:47,038 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:47,039 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2533, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:47,039 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2533, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,039 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,044 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:47,044 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=108.0K (856/sec) 2020-05-21 17:28:47,044 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2533, ppid=2532, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,051 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=428 2020-05-21 17:28:47,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:28:47,051 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=108.4K (859/sec) 2020-05-21 17:28:47,051 TRACE [PEWorker-10] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:47,051 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,056 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=401 2020-05-21 17:28:47,056 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=108.8K (862/sec) 2020-05-21 17:28:47,056 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE has lock 2020-05-21 17:28:47,056 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished subprocedure pid=2533, resume processing parent pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,056 INFO [PEWorker-10] procedure2.ProcedureExecutor: Finished pid=2533, ppid=2532, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 308msec 2020-05-21 17:28:47,056 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2533 runningCount=16, activeCount=0 2020-05-21 17:28:47,056 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,056 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2532 runningCount=1, activeCount=1 2020-05-21 17:28:47,057 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; cycles=0 2020-05-21 17:28:47,057 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,057 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2532 2020-05-21 17:28:47,057 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:47,057 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,057 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,062 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=405 2020-05-21 17:28:47,062 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=109.2K (865/sec) 2020-05-21 17:28:47,062 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; cycles=0 2020-05-21 17:28:47,062 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,062 INFO [PEWorker-10] assignment.TransitRegionStateProcedure: Starting pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:47,062 TRACE [PEWorker-10] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=false, [] 2020-05-21 17:28:47,062 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Suspend pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,062 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:47,062 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,062 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,067 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=409 2020-05-21 17:28:47,067 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=109.6K (868/sec) 2020-05-21 17:28:47,067 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2532 runningCount=16, activeCount=0 2020-05-21 17:28:47,212 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:47,212 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:47,213 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:47,213 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:47,213 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=true, [pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE] 2020-05-21 17:28:47,213 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,213 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE has lock 2020-05-21 17:28:47,213 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:47,214 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,214 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2532 runningCount=1, activeCount=1 2020-05-21 17:28:47,214 TRACE [PEWorker-6] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; cycles=0 2020-05-21 17:28:47,214 TRACE [PEWorker-6] procedure2.StateMachineProcedure: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,214 INFO [PEWorker-6] assignment.RegionStateStore: pid=2532 updating hbase:meta row=20404052b3c0674a71c9f9da299b9e17, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:47,217 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 62, totalSize: 6 bytes 2020-05-21 17:28:47,217 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:47,217 TRACE [PEWorker-6] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:47,217 INFO [PEWorker-6] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:47,217 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:47,217 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Stored pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE, children [pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:47,218 TRACE [PEWorker-6] wal.WALProcedureStore: Insert pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE, subproc=[pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:47,223 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=757 2020-05-21 17:28:47,223 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=110.3K (873/sec) 2020-05-21 17:28:47,224 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,224 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2532 runningCount=16, activeCount=0 2020-05-21 17:28:47,224 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,224 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2534 runningCount=1, activeCount=1 2020-05-21 17:28:47,224 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,229 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:47,229 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=110.7K (876/sec) 2020-05-21 17:28:47,230 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=false, [] 2020-05-21 17:28:47,230 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,230 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:47,230 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,230 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,236 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=353 2020-05-21 17:28:47,236 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=111.0K (878/sec) 2020-05-21 17:28:47,236 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,243 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=350 2020-05-21 17:28:47,243 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=111.3K (881/sec) 2020-05-21 17:28:47,243 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2534 runningCount=16, activeCount=0 2020-05-21 17:28:47,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726602213224391 ns. 2020-05-21 17:28:47,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 93390 ns. 2020-05-21 17:28:47,380 TRACE [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:47,387 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 63, totalSize: 4 bytes 2020-05-21 17:28:47,387 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 7ms 2020-05-21 17:28:47,405 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 511 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 167 bytes 2020-05-21 17:28:47,405 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 511 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 167 connection: 10.128.53.53:44503 deadline: 1590082137405 executing as hbase 2020-05-21 17:28:47,406 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17 regionState=OPENED 2020-05-21 17:28:47,406 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,506 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=111.3K (879/sec) 2020-05-21 17:28:47,512 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=355 2020-05-21 17:28:47,512 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=111.7K (882/sec) 2020-05-21 17:28:47,513 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 20404052b3c0674a71c9f9da299b9e17, NAME => 'aeris_test_backup_restore,,1581768003353.20404052b3c0674a71c9f9da299b9e17.', STARTKEY => '', ENDKEY => '\x01'}, ready=true, [pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:47,513 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,513 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,513 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 511 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 167 connection: 10.128.53.53:44503 deadline: 1590082137405 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:47,514 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2534, ppid=2532, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,514 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2534 runningCount=1, activeCount=1 2020-05-21 17:28:47,514 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,524 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=358 2020-05-21 17:28:47,524 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=112.0K (885/sec) 2020-05-21 17:28:47,525 INFO [PEWorker-15] assignment.RegionStateStore: pid=2532 updating hbase:meta row=20404052b3c0674a71c9f9da299b9e17, regionState=OPEN, openSeqNum=243, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:47,527 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 64, totalSize: 6 bytes 2020-05-21 17:28:47,528 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:47,528 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2534, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:47,528 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2534, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,528 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,533 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=360 2020-05-21 17:28:47,533 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=112.4K (887/sec) 2020-05-21 17:28:47,533 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2534, ppid=2532, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:47,539 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=357 2020-05-21 17:28:47,539 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=112.7K (890/sec) 2020-05-21 17:28:47,539 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2532, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:47,540 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,544 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:47,544 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=113.1K (893/sec) 2020-05-21 17:28:47,545 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE has lock 2020-05-21 17:28:47,545 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished subprocedure pid=2534, resume processing parent pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,545 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,545 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished pid=2534, ppid=2532, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 311msec 2020-05-21 17:28:47,545 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2532 runningCount=2, activeCount=2 2020-05-21 17:28:47,545 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2534 runningCount=1, activeCount=1 2020-05-21 17:28:47,545 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE; cycles=0 2020-05-21 17:28:47,545 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2532, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,545 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2532, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:47,545 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2532, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,545 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2532, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE and Delete [2533, 2534] 2020-05-21 17:28:47,551 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:47,551 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=113.6K (896/sec) 2020-05-21 17:28:47,551 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2532, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE 2020-05-21 17:28:47,556 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=418 2020-05-21 17:28:47,556 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=114.0K (900/sec) 2020-05-21 17:28:47,556 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2532, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE released the shared lock 2020-05-21 17:28:47,556 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2532, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=20404052b3c0674a71c9f9da299b9e17, REOPEN/MOVE in 940msec 2020-05-21 17:28:47,557 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2532 runningCount=16, activeCount=0 2020-05-21 17:28:47,614 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=a6bc10ee35b217018a9e0d67385305e9, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:47,614 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE, subproc=null 2020-05-21 17:28:47,715 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=114.0K (899/sec) 2020-05-21 17:28:47,721 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:47,721 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=114.4K (901/sec) 2020-05-21 17:28:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,722 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,723 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2535 runningCount=1, activeCount=1 2020-05-21 17:28:47,723 INFO [PEWorker-8] procedure.MasterProcedureScheduler: Took xlock for pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,723 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,728 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:47,728 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=114.7K (904/sec) 2020-05-21 17:28:47,728 TRACE [PEWorker-8] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; cycles=0 2020-05-21 17:28:47,728 TRACE [PEWorker-8] procedure2.StateMachineProcedure: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:47,728 INFO [PEWorker-8] assignment.RegionStateStore: pid=2535 updating hbase:meta row=a6bc10ee35b217018a9e0d67385305e9, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:47,731 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 65, totalSize: 6 bytes 2020-05-21 17:28:47,732 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:47,732 TRACE [PEWorker-8] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:47,732 INFO [PEWorker-8] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:47,732 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:47,732 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Stored pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE, children [pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:47,732 TRACE [PEWorker-8] wal.WALProcedureStore: Insert pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE, subproc=[pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:47,738 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:47,738 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=115.5K (911/sec) 2020-05-21 17:28:47,738 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,738 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2535 runningCount=16, activeCount=0 2020-05-21 17:28:47,738 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,738 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2536 runningCount=1, activeCount=1 2020-05-21 17:28:47,738 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,745 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:47,745 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=115.9K (914/sec) 2020-05-21 17:28:47,745 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=false, [] 2020-05-21 17:28:47,745 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,745 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:47,745 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,745 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,752 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:47,752 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=116.3K (917/sec) 2020-05-21 17:28:47,752 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:47,757 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:47,757 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=116.7K (920/sec) 2020-05-21 17:28:47,758 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2536 runningCount=16, activeCount=0 2020-05-21 17:28:47,896 TRACE [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:47,897 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 66, totalSize: 4 bytes 2020-05-21 17:28:47,897 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:47,918 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 515 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:47,919 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 515 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082137918 executing as hbase 2020-05-21 17:28:47,919 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9 regionState=CLOSED 2020-05-21 17:28:47,919 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,020 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=116.7K (918/sec) 2020-05-21 17:28:48,025 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:48,025 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=117.2K (921/sec) 2020-05-21 17:28:48,026 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=true, [pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:48,026 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,026 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,027 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:48,027 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2536, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,027 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2536 runningCount=1, activeCount=1 2020-05-21 17:28:48,027 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,027 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 515 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.9.37:50537 deadline: 1590082137918 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 109 totalTime: 109 2020-05-21 17:28:48,033 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:48,033 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=117.6K (925/sec) 2020-05-21 17:28:48,033 INFO [PEWorker-1] assignment.RegionStateStore: pid=2535 updating hbase:meta row=a6bc10ee35b217018a9e0d67385305e9, regionState=CLOSED 2020-05-21 17:28:48,038 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 67, totalSize: 6 bytes 2020-05-21 17:28:48,038 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:48,038 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2536, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:48,038 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2536, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,038 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,045 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:48,045 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=118.0K (928/sec) 2020-05-21 17:28:48,045 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2536, ppid=2535, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,050 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:48,050 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=118.4K (931/sec) 2020-05-21 17:28:48,050 TRACE [PEWorker-1] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:48,051 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,055 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:48,055 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=118.8K (934/sec) 2020-05-21 17:28:48,055 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE has lock 2020-05-21 17:28:48,056 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished subprocedure pid=2536, resume processing parent pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,056 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished pid=2536, ppid=2535, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 306msec 2020-05-21 17:28:48,056 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2536 runningCount=16, activeCount=0 2020-05-21 17:28:48,056 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,056 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2535 runningCount=1, activeCount=1 2020-05-21 17:28:48,056 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; cycles=0 2020-05-21 17:28:48,056 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,056 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2535 2020-05-21 17:28:48,056 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:48,056 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,056 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,061 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:48,061 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=119.2K (937/sec) 2020-05-21 17:28:48,061 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; cycles=0 2020-05-21 17:28:48,061 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,061 INFO [PEWorker-5] assignment.TransitRegionStateProcedure: Starting pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:48,061 TRACE [PEWorker-5] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=false, [] 2020-05-21 17:28:48,061 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Suspend pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,062 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:48,062 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,062 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,065 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 516 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3566 bytes 2020-05-21 17:28:48,065 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 516 service: RegionServerStatusService methodName: RegionServerReport size: 3.5 K connection: 10.128.9.37:50537 deadline: 1590082138065 executing as hbase 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142836614, completeSequenceId=142840890 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98018867 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:48,066 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433055.9bf8f6c9ac2972f06898c86000e5f79f., aeris_v2,\x08\xCD\x02H\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:48,067 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 516 service: RegionServerStatusService methodName: RegionServerReport size: 3.5 K connection: 10.128.9.37:50537 deadline: 1590082138065 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 411 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:48,068 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:48,068 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=119.6K (940/sec) 2020-05-21 17:28:48,068 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2535 runningCount=16, activeCount=0 2020-05-21 17:28:48,157 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 512 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4657 bytes 2020-05-21 17:28:48,159 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 512 service: RegionServerStatusService methodName: RegionServerReport size: 4.5 K connection: 10.128.53.53:44503 deadline: 1590082138157 executing as hbase 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116811182, completeSequenceId=116813585 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,160 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 57bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:48,161 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 512 service: RegionServerStatusService methodName: RegionServerReport size: 4.5 K connection: 10.128.53.53:44503 deadline: 1590082138157 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 420 } connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 3 totalTime: 4 2020-05-21 17:28:48,212 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:48,212 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:48,212 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:48,217 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:48,217 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=true, [pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE] 2020-05-21 17:28:48,217 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,217 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE has lock 2020-05-21 17:28:48,217 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:48,219 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,219 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2535 runningCount=1, activeCount=1 2020-05-21 17:28:48,219 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; cycles=0 2020-05-21 17:28:48,219 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,219 INFO [PEWorker-12] assignment.RegionStateStore: pid=2535 updating hbase:meta row=a6bc10ee35b217018a9e0d67385305e9, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:48,222 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 68, totalSize: 6 bytes 2020-05-21 17:28:48,222 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:48,222 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:48,222 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:48,222 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:48,222 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE, children [pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:48,222 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE, subproc=[pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:48,230 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:48,230 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=120.4K (945/sec) 2020-05-21 17:28:48,230 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,230 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2535 runningCount=16, activeCount=0 2020-05-21 17:28:48,231 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,231 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2537 runningCount=1, activeCount=1 2020-05-21 17:28:48,231 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,235 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:48,235 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=120.7K (948/sec) 2020-05-21 17:28:48,236 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=false, [] 2020-05-21 17:28:48,236 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,236 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:48,236 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,236 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,241 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:48,241 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=121.0K (950/sec) 2020-05-21 17:28:48,241 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,247 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:48,247 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=121.4K (953/sec) 2020-05-21 17:28:48,247 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2537 runningCount=16, activeCount=0 2020-05-21 17:28:48,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726603213203923 ns. 2020-05-21 17:28:48,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 106402 ns. 2020-05-21 17:28:48,386 TRACE [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:48,392 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 69, totalSize: 4 bytes 2020-05-21 17:28:48,392 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:28:48,419 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 517 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:48,420 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 517 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082138420 executing as hbase 2020-05-21 17:28:48,420 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9 regionState=OPENED 2020-05-21 17:28:48,420 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,521 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=121.4K (951/sec) 2020-05-21 17:28:48,526 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=356 2020-05-21 17:28:48,526 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=121.7K (954/sec) 2020-05-21 17:28:48,527 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => a6bc10ee35b217018a9e0d67385305e9, NAME => 'aeris_test_backup_restore,\x03,1581768003353.a6bc10ee35b217018a9e0d67385305e9.', STARTKEY => '\x03', ENDKEY => '\x04'}, ready=true, [pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:48,527 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,527 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,527 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 517 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.9.37:50537 deadline: 1590082138420 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 107 totalTime: 107 2020-05-21 17:28:48,527 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2537, ppid=2535, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,527 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2537 runningCount=1, activeCount=1 2020-05-21 17:28:48,527 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,532 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=359 2020-05-21 17:28:48,532 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=122.1K (956/sec) 2020-05-21 17:28:48,533 INFO [PEWorker-14] assignment.RegionStateStore: pid=2535 updating hbase:meta row=a6bc10ee35b217018a9e0d67385305e9, regionState=OPEN, openSeqNum=295, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:48,535 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 70, totalSize: 6 bytes 2020-05-21 17:28:48,535 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:48,536 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2537, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:48,536 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2537, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,536 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,541 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=361 2020-05-21 17:28:48,541 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=122.4K (959/sec) 2020-05-21 17:28:48,541 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2537, ppid=2535, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:48,546 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=358 2020-05-21 17:28:48,546 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=122.8K (962/sec) 2020-05-21 17:28:48,547 TRACE [PEWorker-14] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2535, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:48,547 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:48,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=123.2K (965/sec) 2020-05-21 17:28:48,554 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE has lock 2020-05-21 17:28:48,554 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished subprocedure pid=2537, resume processing parent pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,554 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2537, ppid=2535, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 314msec 2020-05-21 17:28:48,554 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,554 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2537 runningCount=16, activeCount=0 2020-05-21 17:28:48,554 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Execute pid=2535 runningCount=1, activeCount=1 2020-05-21 17:28:48,554 TRACE [PEWorker-7] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE; cycles=0 2020-05-21 17:28:48,554 TRACE [PEWorker-7] procedure2.StateMachineProcedure: pid=2535, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,554 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2535, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:48,554 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2535, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,554 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2535, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE and Delete [2536, 2537] 2020-05-21 17:28:48,559 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:48,559 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=123.6K (968/sec) 2020-05-21 17:28:48,559 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2535, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE 2020-05-21 17:28:48,565 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=419 2020-05-21 17:28:48,565 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=124.0K (971/sec) 2020-05-21 17:28:48,566 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2535, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE released the shared lock 2020-05-21 17:28:48,566 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished pid=2535, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=a6bc10ee35b217018a9e0d67385305e9, REOPEN/MOVE in 940msec 2020-05-21 17:28:48,566 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Halt pid=2535 runningCount=16, activeCount=0 2020-05-21 17:28:48,623 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=10c81f35e28121d9c1f3f730c673728b, source=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:48,623 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE, subproc=null 2020-05-21 17:28:48,724 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=124.0K (970/sec) 2020-05-21 17:28:48,729 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=504 2020-05-21 17:28:48,729 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=124.5K (974/sec) 2020-05-21 17:28:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,732 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,732 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2538 runningCount=1, activeCount=1 2020-05-21 17:28:48,732 INFO [PEWorker-16] procedure.MasterProcedureScheduler: Took xlock for pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,732 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,746 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=507 2020-05-21 17:28:48,746 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=125.0K (978/sec) 2020-05-21 17:28:48,746 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; cycles=0 2020-05-21 17:28:48,746 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:48,746 INFO [PEWorker-16] assignment.RegionStateStore: pid=2538 updating hbase:meta row=10c81f35e28121d9c1f3f730c673728b, regionState=CLOSING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:48,748 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 71, totalSize: 6 bytes 2020-05-21 17:28:48,749 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:48,749 TRACE [PEWorker-16] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:48,749 INFO [PEWorker-16] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:48,749 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:48,749 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Stored pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE, children [pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:48,749 TRACE [PEWorker-16] wal.WALProcedureStore: Insert pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE, subproc=[pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:48,754 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=1.0K 2020-05-21 17:28:48,754 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=126.0K (986/sec) 2020-05-21 17:28:48,754 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,754 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2538 runningCount=16, activeCount=0 2020-05-21 17:28:48,755 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,755 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2539 runningCount=1, activeCount=1 2020-05-21 17:28:48,755 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,760 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=527 2020-05-21 17:28:48,760 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=126.5K (990/sec) 2020-05-21 17:28:48,760 TRACE [PEWorker-2] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:48,760 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Suspend pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,760 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:48,760 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Store update pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,760 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=529 2020-05-21 17:28:48,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=127.1K (994/sec) 2020-05-21 17:28:48,766 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:48,771 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=526 2020-05-21 17:28:48,771 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=127.6K (998/sec) 2020-05-21 17:28:48,771 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2539 runningCount=16, activeCount=0 2020-05-21 17:28:48,911 TRACE [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:48,913 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 72, totalSize: 4 bytes 2020-05-21 17:28:48,913 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:49,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726604213268268 ns. 2020-05-21 17:28:49,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 186119 ns. 2020-05-21 17:28:49,464 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 518 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 279 bytes 2020-05-21 17:28:49,464 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 518 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 279 connection: 10.128.9.37:50537 deadline: 1590082139464 executing as hbase 2020-05-21 17:28:49,465 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=CLOSING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b regionState=CLOSED 2020-05-21 17:28:49,465 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,566 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=127.6K (992/sec) 2020-05-21 17:28:49,573 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=539 2020-05-21 17:28:49,573 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=128.1K (996/sec) 2020-05-21 17:28:49,573 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:49,574 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,574 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,574 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:49,574 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 518 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 279 connection: 10.128.9.37:50537 deadline: 1590082139464 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 110 totalTime: 110 2020-05-21 17:28:49,574 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082129573, delay=0 2020-05-21 17:28:49,575 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2539, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,575 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2539 runningCount=1, activeCount=1 2020-05-21 17:28:49,575 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,580 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=542 2020-05-21 17:28:49,580 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=128.6K (1000/sec) 2020-05-21 17:28:49,580 INFO [PEWorker-9] assignment.RegionStateStore: pid=2538 updating hbase:meta row=10c81f35e28121d9c1f3f730c673728b, regionState=CLOSED 2020-05-21 17:28:49,582 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 73, totalSize: 6 bytes 2020-05-21 17:28:49,582 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:49,583 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2539, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:49,583 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2539, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,583 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,588 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=544 2020-05-21 17:28:49,588 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=129.2K (1004/sec) 2020-05-21 17:28:49,588 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2539, ppid=2538, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:49,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=541 2020-05-21 17:28:49,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=129.7K (1008/sec) 2020-05-21 17:28:49,593 TRACE [PEWorker-9] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:49,593 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,599 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=515 2020-05-21 17:28:49,599 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=130.2K (1012/sec) 2020-05-21 17:28:49,599 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE has lock 2020-05-21 17:28:49,599 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished subprocedure pid=2539, resume processing parent pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,599 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2539, ppid=2538, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 833msec 2020-05-21 17:28:49,599 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2539 runningCount=16, activeCount=0 2020-05-21 17:28:49,599 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,599 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2538 runningCount=1, activeCount=1 2020-05-21 17:28:49,599 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; cycles=0 2020-05-21 17:28:49,599 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,599 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2538 2020-05-21 17:28:49,599 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:49,599 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,599 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,604 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=519 2020-05-21 17:28:49,604 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=130.7K (1016/sec) 2020-05-21 17:28:49,604 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; cycles=0 2020-05-21 17:28:49,604 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,604 INFO [PEWorker-10] assignment.TransitRegionStateProcedure: Starting pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:49,604 TRACE [PEWorker-10] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:49,604 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Suspend pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,604 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:49,604 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,604 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,609 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=523 2020-05-21 17:28:49,609 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=131.2K (1019/sec) 2020-05-21 17:28:49,609 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2538 runningCount=16, activeCount=0 2020-05-21 17:28:49,754 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:49,754 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:49,754 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:49,755 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:49,755 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE] 2020-05-21 17:28:49,755 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,755 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE has lock 2020-05-21 17:28:49,755 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:49,756 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,756 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2538 runningCount=1, activeCount=1 2020-05-21 17:28:49,756 TRACE [PEWorker-3] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; cycles=0 2020-05-21 17:28:49,756 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:49,756 INFO [PEWorker-3] assignment.RegionStateStore: pid=2538 updating hbase:meta row=10c81f35e28121d9c1f3f730c673728b, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:49,759 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 74, totalSize: 6 bytes 2020-05-21 17:28:49,759 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:49,759 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:49,759 INFO [PEWorker-3] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:49,759 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:49,759 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Stored pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE, children [pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:49,759 TRACE [PEWorker-3] wal.WALProcedureStore: Insert pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE, subproc=[pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:49,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=985 2020-05-21 17:28:49,767 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=132.2K (1.0K/sec) 2020-05-21 17:28:49,767 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,767 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2538 runningCount=16, activeCount=0 2020-05-21 17:28:49,767 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,767 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2540 runningCount=1, activeCount=1 2020-05-21 17:28:49,767 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,773 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=465 2020-05-21 17:28:49,773 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=132.6K (1.0K/sec) 2020-05-21 17:28:49,773 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:49,773 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,773 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:49,773 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,773 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=467 2020-05-21 17:28:49,778 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=133.1K (1.0K/sec) 2020-05-21 17:28:49,778 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:49,783 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=464 2020-05-21 17:28:49,783 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=133.5K (1.0K/sec) 2020-05-21 17:28:49,783 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2540 runningCount=16, activeCount=0 2020-05-21 17:28:49,924 TRACE [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:49,930 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 75, totalSize: 4 bytes 2020-05-21 17:28:49,930 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:28:49,974 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 519 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 284 bytes 2020-05-21 17:28:49,974 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 519 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 284 connection: 10.128.9.37:50537 deadline: 1590082139974 executing as hbase 2020-05-21 17:28:49,975 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b regionState=OPENED 2020-05-21 17:28:49,975 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,076 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=133.5K (1.0K/sec) 2020-05-21 17:28:50,082 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=471 2020-05-21 17:28:50,082 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=134.0K (1.0K/sec) 2020-05-21 17:28:50,082 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 10c81f35e28121d9c1f3f730c673728b, NAME => 'aeris_v2,\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1590081438310.10c81f35e28121d9c1f3f730c673728b.', STARTKEY => '\x06\xCD\x01\xF5\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x06\xCD\x02:\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:50,082 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,082 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,082 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 519 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 284 connection: 10.128.9.37:50537 deadline: 1590082139974 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:50,082 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2540, ppid=2538, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,082 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2540 runningCount=1, activeCount=1 2020-05-21 17:28:50,083 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,088 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=474 2020-05-21 17:28:50,088 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=134.5K (1.0K/sec) 2020-05-21 17:28:50,088 INFO [PEWorker-15] assignment.RegionStateStore: pid=2538 updating hbase:meta row=10c81f35e28121d9c1f3f730c673728b, regionState=OPEN, openSeqNum=98020794, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:50,091 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 76, totalSize: 6 bytes 2020-05-21 17:28:50,091 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:50,092 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2540, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:50,092 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2540, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,092 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,099 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=476 2020-05-21 17:28:50,099 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=134.9K (1.0K/sec) 2020-05-21 17:28:50,099 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2540, ppid=2538, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=473 2020-05-21 17:28:50,104 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=135.4K (1.0K/sec) 2020-05-21 17:28:50,105 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2538, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:50,105 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,111 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=527 2020-05-21 17:28:50,111 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=135.9K (1.0K/sec) 2020-05-21 17:28:50,111 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE has lock 2020-05-21 17:28:50,111 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished subprocedure pid=2540, resume processing parent pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,111 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,111 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished pid=2540, ppid=2538, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 333msec 2020-05-21 17:28:50,111 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2538 runningCount=2, activeCount=2 2020-05-21 17:28:50,111 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2540 runningCount=1, activeCount=1 2020-05-21 17:28:50,111 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE; cycles=0 2020-05-21 17:28:50,111 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2538, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,112 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2538, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:50,112 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2538, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,112 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2538, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE and Delete [2540, 2539] 2020-05-21 17:28:50,119 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=544 2020-05-21 17:28:50,119 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=136.4K (1.0K/sec) 2020-05-21 17:28:50,119 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2538, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE 2020-05-21 17:28:50,124 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=532 2020-05-21 17:28:50,124 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=136.9K (1.0K/sec) 2020-05-21 17:28:50,124 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2538, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE released the shared lock 2020-05-21 17:28:50,124 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2538, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=10c81f35e28121d9c1f3f730c673728b, REOPEN/MOVE in 1.4890sec 2020-05-21 17:28:50,124 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2538 runningCount=16, activeCount=0 2020-05-21 17:28:50,132 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=f4e57ea829d51cdcb1c8d74a938311c2, source=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:50,132 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE, subproc=null 2020-05-21 17:28:50,233 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=136.9K (1.0K/sec) 2020-05-21 17:28:50,239 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=515 2020-05-21 17:28:50,239 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=137.4K (1.0K/sec) 2020-05-21 17:28:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,240 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,240 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2541 runningCount=1, activeCount=1 2020-05-21 17:28:50,240 INFO [PEWorker-11] procedure.MasterProcedureScheduler: Took xlock for pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,240 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,246 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=518 2020-05-21 17:28:50,246 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=138.0K (1.0K/sec) 2020-05-21 17:28:50,247 TRACE [PEWorker-11] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; cycles=0 2020-05-21 17:28:50,247 TRACE [PEWorker-11] procedure2.StateMachineProcedure: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,247 INFO [PEWorker-11] assignment.RegionStateStore: pid=2541 updating hbase:meta row=f4e57ea829d51cdcb1c8d74a938311c2, regionState=CLOSING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:50,250 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 77, totalSize: 6 bytes 2020-05-21 17:28:50,250 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:50,250 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:50,250 INFO [PEWorker-11] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:50,250 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:50,250 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Stored pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE, children [pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:50,250 TRACE [PEWorker-11] wal.WALProcedureStore: Insert pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE, subproc=[pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:50,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726605213262324 ns. 2020-05-21 17:28:50,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 81635 ns. 2020-05-21 17:28:50,256 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=1.0K 2020-05-21 17:28:50,256 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=139.0K (1.0K/sec) 2020-05-21 17:28:50,257 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,257 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2541 runningCount=16, activeCount=0 2020-05-21 17:28:50,257 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,257 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2542 runningCount=1, activeCount=1 2020-05-21 17:28:50,257 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,263 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=538 2020-05-21 17:28:50,263 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=139.5K (1.1K/sec) 2020-05-21 17:28:50,263 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:50,263 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,263 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:50,263 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,263 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,270 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=540 2020-05-21 17:28:50,270 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=140.0K (1.1K/sec) 2020-05-21 17:28:50,270 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,277 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=537 2020-05-21 17:28:50,277 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=140.6K (1.1K/sec) 2020-05-21 17:28:50,277 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2542 runningCount=16, activeCount=0 2020-05-21 17:28:50,414 TRACE [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:50,416 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 78, totalSize: 4 bytes 2020-05-21 17:28:50,416 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:50,436 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 513 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 290 bytes 2020-05-21 17:28:50,436 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 513 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 290 connection: 10.128.53.53:44503 deadline: 1590082140436 executing as hbase 2020-05-21 17:28:50,437 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=CLOSING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2 regionState=CLOSED 2020-05-21 17:28:50,437 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,537 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=140.6K (1.1K/sec) 2020-05-21 17:28:50,543 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=550 2020-05-21 17:28:50,544 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=141.1K (1.1K/sec) 2020-05-21 17:28:50,544 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:50,544 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,545 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,545 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:50,545 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 513 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 290 connection: 10.128.53.53:44503 deadline: 1590082140436 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 109 totalTime: 109 2020-05-21 17:28:50,545 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2542, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,545 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2542 runningCount=1, activeCount=1 2020-05-21 17:28:50,546 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=553 2020-05-21 17:28:50,553 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=141.6K (1.1K/sec) 2020-05-21 17:28:50,554 INFO [PEWorker-1] assignment.RegionStateStore: pid=2541 updating hbase:meta row=f4e57ea829d51cdcb1c8d74a938311c2, regionState=CLOSED 2020-05-21 17:28:50,556 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 79, totalSize: 6 bytes 2020-05-21 17:28:50,556 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:50,556 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2542, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:50,556 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2542, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,556 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,567 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=555 2020-05-21 17:28:50,567 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=142.2K (1.1K/sec) 2020-05-21 17:28:50,567 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2542, ppid=2541, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:50,572 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=552 2020-05-21 17:28:50,572 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=142.7K (1.1K/sec) 2020-05-21 17:28:50,573 TRACE [PEWorker-1] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:50,573 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,578 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=526 2020-05-21 17:28:50,578 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=143.2K (1.1K/sec) 2020-05-21 17:28:50,578 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE has lock 2020-05-21 17:28:50,578 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished subprocedure pid=2542, resume processing parent pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,578 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished pid=2542, ppid=2541, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 306msec 2020-05-21 17:28:50,578 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,578 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2542 runningCount=16, activeCount=0 2020-05-21 17:28:50,578 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2541 runningCount=1, activeCount=1 2020-05-21 17:28:50,578 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; cycles=0 2020-05-21 17:28:50,578 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,578 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2541 2020-05-21 17:28:50,578 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:50,578 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,578 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,583 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=530 2020-05-21 17:28:50,583 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=143.8K (1.1K/sec) 2020-05-21 17:28:50,583 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; cycles=0 2020-05-21 17:28:50,584 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,584 INFO [PEWorker-5] assignment.TransitRegionStateProcedure: Starting pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:50,584 TRACE [PEWorker-5] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:50,584 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Suspend pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,584 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:50,584 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,584 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,591 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=534 2020-05-21 17:28:50,591 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=144.3K (1.1K/sec) 2020-05-21 17:28:50,591 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2541 runningCount=16, activeCount=0 2020-05-21 17:28:50,734 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:50,734 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:50,734 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:50,735 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:50,735 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE] 2020-05-21 17:28:50,735 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,735 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE has lock 2020-05-21 17:28:50,735 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:50,735 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,735 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2541 runningCount=1, activeCount=1 2020-05-21 17:28:50,735 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; cycles=0 2020-05-21 17:28:50,735 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:50,735 INFO [PEWorker-12] assignment.RegionStateStore: pid=2541 updating hbase:meta row=f4e57ea829d51cdcb1c8d74a938311c2, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:50,738 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 80, totalSize: 6 bytes 2020-05-21 17:28:50,738 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:50,738 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:50,738 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:50,738 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:50,738 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE, children [pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:50,738 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE, subproc=[pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:50,746 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=1007 2020-05-21 17:28:50,746 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=145.3K (1.1K/sec) 2020-05-21 17:28:50,746 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,746 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2541 runningCount=16, activeCount=0 2020-05-21 17:28:50,746 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,746 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2543 runningCount=1, activeCount=1 2020-05-21 17:28:50,746 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,755 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=476 2020-05-21 17:28:50,755 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=145.7K (1.1K/sec) 2020-05-21 17:28:50,755 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:50,755 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,755 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:50,755 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,755 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,761 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=478 2020-05-21 17:28:50,761 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=146.2K (1.1K/sec) 2020-05-21 17:28:50,761 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:50,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=475 2020-05-21 17:28:50,766 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=146.7K (1.1K/sec) 2020-05-21 17:28:50,766 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2543 runningCount=16, activeCount=0 2020-05-21 17:28:50,906 TRACE [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:50,911 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 81, totalSize: 4 bytes 2020-05-21 17:28:50,912 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:28:50,947 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 520 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 295 bytes 2020-05-21 17:28:50,947 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 520 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 295 connection: 10.128.9.37:50537 deadline: 1590082140947 executing as hbase 2020-05-21 17:28:50,948 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2 regionState=OPENED 2020-05-21 17:28:50,948 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,048 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=146.7K (1.1K/sec) 2020-05-21 17:28:51,054 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=482 2020-05-21 17:28:51,054 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=147.1K (1.1K/sec) 2020-05-21 17:28:51,054 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => f4e57ea829d51cdcb1c8d74a938311c2, NAME => 'aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2.', STARTKEY => '\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24', ENDKEY => '\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:51,054 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,054 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,054 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2543, ppid=2541, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,054 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 520 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 295 connection: 10.128.9.37:50537 deadline: 1590082140947 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 107 totalTime: 107 2020-05-21 17:28:51,054 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2543 runningCount=1, activeCount=1 2020-05-21 17:28:51,055 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,060 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=485 2020-05-21 17:28:51,060 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=147.6K (1.1K/sec) 2020-05-21 17:28:51,060 INFO [PEWorker-14] assignment.RegionStateStore: pid=2541 updating hbase:meta row=f4e57ea829d51cdcb1c8d74a938311c2, regionState=OPEN, openSeqNum=98729328, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:51,063 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 82, totalSize: 6 bytes 2020-05-21 17:28:51,063 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:51,063 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2543, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:51,064 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2543, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,064 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,069 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 521 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 3930 bytes 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 521 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.9.37:50537 deadline: 1590082141069 executing as hbase 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98018867, completeSequenceId=98020795 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:51,070 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: EStrataCounters\xC4\x07current\xC4\x07counter,1587075433456.86533e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:51,071 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 521 service: RegionServerStatusService methodName: RegionServerReport size: 3.8 K connection: 10.128.9.37:50537 deadline: 1590082141069 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 411 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:51,073 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=487 2020-05-21 17:28:51,073 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=148.1K (1.1K/sec) 2020-05-21 17:28:51,073 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2543, ppid=2541, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:51,078 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=484 2020-05-21 17:28:51,078 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=148.5K (1.1K/sec) 2020-05-21 17:28:51,079 TRACE [PEWorker-14] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2541, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:51,079 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,084 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=538 2020-05-21 17:28:51,084 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=149.1K (1.1K/sec) 2020-05-21 17:28:51,084 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE has lock 2020-05-21 17:28:51,084 DEBUG [PEWorker-7] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,084 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Execute pid=2541 runningCount=2, activeCount=2 2020-05-21 17:28:51,084 TRACE [PEWorker-7] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE; cycles=0 2020-05-21 17:28:51,084 TRACE [PEWorker-7] procedure2.StateMachineProcedure: pid=2541, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,084 DEBUG [PEWorker-7] procedure2.RootProcedureState: Add procedure pid=2541, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:51,084 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Store update pid=2541, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,084 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2541, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE and Delete [2543, 2542] 2020-05-21 17:28:51,085 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished subprocedure pid=2543, resume processing parent pid=2541, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,085 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2543, ppid=2541, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 325msec 2020-05-21 17:28:51,085 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2543 runningCount=1, activeCount=1 2020-05-21 17:28:51,140 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=e855ef99e11ec78bead3c9e21e6da12f, source=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:51,141 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE, subproc=null 2020-05-21 17:28:51,141 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 56msec, slotIndex=2 , totalSynced=149.1K (1.1K/sec) 2020-05-21 17:28:51,147 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=2/2, flushed=994 2020-05-21 17:28:51,147 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=150.0K (1.1K/sec) 2020-05-21 17:28:51,148 TRACE [PEWorker-7] wal.WALProcedureStore: Update pid=2541, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE 2020-05-21 17:28:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,151 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,151 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2544 runningCount=2, activeCount=2 2020-05-21 17:28:51,151 INFO [PEWorker-16] procedure.MasterProcedureScheduler: Took xlock for pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,151 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,156 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=543 2020-05-21 17:28:51,157 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=150.6K (1.1K/sec) 2020-05-21 17:28:51,157 INFO [PEWorker-7] procedure2.ProcedureExecutor: Finished pid=2541, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=f4e57ea829d51cdcb1c8d74a938311c2, REOPEN/MOVE in 952msec 2020-05-21 17:28:51,157 TRACE [PEWorker-7] procedure2.ProcedureExecutor: Halt pid=2541 runningCount=1, activeCount=1 2020-05-21 17:28:51,164 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 514 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4432 bytes 2020-05-21 17:28:51,165 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 514 service: RegionServerStatusService methodName: RegionServerReport size: 4.3 K connection: 10.128.53.53:44503 deadline: 1590082141165 executing as hbase 2020-05-21 17:28:51,165 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,165 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,165 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121128147 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90599702 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144617, completeSequenceId=80144966 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 57bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:51,166 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 514 service: RegionServerStatusService methodName: RegionServerReport size: 4.3 K connection: 10.128.53.53:44503 deadline: 1590082141165 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 420 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:51,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726606213250994 ns. 2020-05-21 17:28:51,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 95166 ns. 2020-05-21 17:28:51,257 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=150.6K (1.1K/sec) 2020-05-21 17:28:51,264 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/2, flushed=442 2020-05-21 17:28:51,264 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=151.0K (1.1K/sec) 2020-05-21 17:28:51,264 TRACE [PEWorker-16] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:51,264 TRACE [PEWorker-16] procedure2.StateMachineProcedure: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,264 INFO [PEWorker-16] assignment.RegionStateStore: pid=2544 updating hbase:meta row=e855ef99e11ec78bead3c9e21e6da12f, regionState=CLOSING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:51,266 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 83, totalSize: 6 bytes 2020-05-21 17:28:51,267 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:51,267 TRACE [PEWorker-16] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:51,267 INFO [PEWorker-16] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:51,267 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:51,267 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Stored pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE, children [pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:51,267 TRACE [PEWorker-16] wal.WALProcedureStore: Insert pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE, subproc=[pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:51,274 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=905 2020-05-21 17:28:51,274 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=151.9K (1.1K/sec) 2020-05-21 17:28:51,274 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,274 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2544 runningCount=16, activeCount=0 2020-05-21 17:28:51,275 DEBUG [PEWorker-16] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,275 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Execute pid=2545 runningCount=1, activeCount=1 2020-05-21 17:28:51,275 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,285 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=462 2020-05-21 17:28:51,285 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=152.3K (1.1K/sec) 2020-05-21 17:28:51,285 TRACE [PEWorker-16] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:51,285 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Suspend pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,285 DEBUG [PEWorker-16] procedure2.RootProcedureState: Add procedure pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:51,285 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Store update pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,285 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,291 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=464 2020-05-21 17:28:51,291 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=152.8K (1.1K/sec) 2020-05-21 17:28:51,291 TRACE [PEWorker-16] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,300 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=461 2020-05-21 17:28:51,300 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=153.2K (1.1K/sec) 2020-05-21 17:28:51,300 TRACE [PEWorker-16] procedure2.ProcedureExecutor: Halt pid=2545 runningCount=16, activeCount=0 2020-05-21 17:28:51,436 TRACE [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:51,438 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 84, totalSize: 4 bytes 2020-05-21 17:28:51,438 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:51,737 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 515 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 213 bytes 2020-05-21 17:28:51,737 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 515 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 213 connection: 10.128.53.53:44503 deadline: 1590082141737 executing as hbase 2020-05-21 17:28:51,737 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=CLOSING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f regionState=CLOSED 2020-05-21 17:28:51,738 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,838 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=153.2K (1.1K/sec) 2020-05-21 17:28:51,844 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=474 2020-05-21 17:28:51,844 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=153.7K (1.1K/sec) 2020-05-21 17:28:51,845 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:51,845 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,845 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,845 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:51,846 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 515 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 213 connection: 10.128.53.53:44503 deadline: 1590082141737 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:51,846 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2545, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,846 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2545 runningCount=1, activeCount=1 2020-05-21 17:28:51,846 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,851 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=477 2020-05-21 17:28:51,851 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=154.2K (1.2K/sec) 2020-05-21 17:28:51,852 INFO [PEWorker-9] assignment.RegionStateStore: pid=2544 updating hbase:meta row=e855ef99e11ec78bead3c9e21e6da12f, regionState=CLOSED 2020-05-21 17:28:51,854 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 85, totalSize: 6 bytes 2020-05-21 17:28:51,854 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:51,854 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2545, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:51,854 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2545, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,854 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,860 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=479 2020-05-21 17:28:51,860 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=154.6K (1.2K/sec) 2020-05-21 17:28:51,860 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2545, ppid=2544, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:51,865 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=476 2020-05-21 17:28:51,865 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=155.1K (1.2K/sec) 2020-05-21 17:28:51,865 TRACE [PEWorker-9] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:51,865 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,870 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=450 2020-05-21 17:28:51,870 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=155.5K (1.2K/sec) 2020-05-21 17:28:51,870 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE has lock 2020-05-21 17:28:51,870 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished subprocedure pid=2545, resume processing parent pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,870 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,870 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2545, ppid=2544, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 587msec 2020-05-21 17:28:51,870 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2544 runningCount=2, activeCount=2 2020-05-21 17:28:51,870 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2545 runningCount=1, activeCount=1 2020-05-21 17:28:51,870 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:51,870 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,870 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2544 2020-05-21 17:28:51,870 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:51,870 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,870 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,878 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=454 2020-05-21 17:28:51,878 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=156.0K (1.2K/sec) 2020-05-21 17:28:51,878 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:51,879 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,879 INFO [PEWorker-10] assignment.TransitRegionStateProcedure: Starting pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:51,879 TRACE [PEWorker-10] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:51,879 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Suspend pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,879 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:51,879 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,879 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:51,885 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=458 2020-05-21 17:28:51,885 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=156.4K (1.2K/sec) 2020-05-21 17:28:51,885 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2544 runningCount=16, activeCount=0 2020-05-21 17:28:52,029 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:52,029 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:52,029 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:52,030 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:52,030 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE] 2020-05-21 17:28:52,030 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,030 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE has lock 2020-05-21 17:28:52,030 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:52,030 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,030 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2544 runningCount=1, activeCount=1 2020-05-21 17:28:52,030 TRACE [PEWorker-3] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:52,030 TRACE [PEWorker-3] procedure2.StateMachineProcedure: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,031 INFO [PEWorker-3] assignment.RegionStateStore: pid=2544 updating hbase:meta row=e855ef99e11ec78bead3c9e21e6da12f, regionState=OPENING, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:52,033 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 86, totalSize: 6 bytes 2020-05-21 17:28:52,033 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:52,034 TRACE [PEWorker-3] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:52,034 INFO [PEWorker-3] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:52,034 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:52,034 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Stored pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE, children [pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:52,034 TRACE [PEWorker-3] wal.WALProcedureStore: Insert pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE, subproc=[pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:52,040 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=855 2020-05-21 17:28:52,040 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=157.3K (1.2K/sec) 2020-05-21 17:28:52,040 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,040 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2544 runningCount=16, activeCount=0 2020-05-21 17:28:52,040 DEBUG [PEWorker-3] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,040 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Execute pid=2546 runningCount=1, activeCount=1 2020-05-21 17:28:52,040 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,047 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=400 2020-05-21 17:28:52,047 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=157.7K (1.2K/sec) 2020-05-21 17:28:52,047 TRACE [PEWorker-3] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=false, [] 2020-05-21 17:28:52,047 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Suspend pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,047 DEBUG [PEWorker-3] procedure2.RootProcedureState: Add procedure pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:52,047 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Store update pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,047 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,054 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:52,054 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=158.0K (1.2K/sec) 2020-05-21 17:28:52,054 TRACE [PEWorker-3] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,059 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=399 2020-05-21 17:28:52,059 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=158.4K (1.2K/sec) 2020-05-21 17:28:52,059 TRACE [PEWorker-3] procedure2.ProcedureExecutor: Halt pid=2546 runningCount=16, activeCount=0 2020-05-21 17:28:52,198 TRACE [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:52,210 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcDuplexHandler: got response header call_id: 87, totalSize: 4 bytes 2020-05-21 17:28:52,210 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 12ms 2020-05-21 17:28:52,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726607213290222 ns. 2020-05-21 17:28:52,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 361933 ns. 2020-05-21 17:28:52,267 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 522 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 218 bytes 2020-05-21 17:28:52,268 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 522 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 218 connection: 10.128.9.37:50537 deadline: 1590082142267 executing as hbase 2020-05-21 17:28:52,268 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 region=rit=OPENING, location=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f regionState=OPENED 2020-05-21 17:28:52,268 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,369 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=158.4K (1.2K/sec) 2020-05-21 17:28:52,375 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=406 2020-05-21 17:28:52,375 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=158.8K (1.2K/sec) 2020-05-21 17:28:52,375 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => e855ef99e11ec78bead3c9e21e6da12f, NAME => 'aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f.', STARTKEY => '\x0F', ENDKEY => '\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter'}, ready=true, [pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:52,375 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,375 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,375 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 522 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 218 connection: 10.128.9.37:50537 deadline: 1590082142267 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:52,376 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2546, ppid=2544, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,376 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2546 runningCount=1, activeCount=1 2020-05-21 17:28:52,376 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=409 2020-05-21 17:28:52,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=159.2K (1.2K/sec) 2020-05-21 17:28:52,382 INFO [PEWorker-15] assignment.RegionStateStore: pid=2544 updating hbase:meta row=e855ef99e11ec78bead3c9e21e6da12f, regionState=OPEN, openSeqNum=90604374, regionLocation=regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754 2020-05-21 17:28:52,385 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 88, totalSize: 6 bytes 2020-05-21 17:28:52,385 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:52,385 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2546, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:52,385 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2546, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,385 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,394 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=411 2020-05-21 17:28:52,394 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=159.6K (1.2K/sec) 2020-05-21 17:28:52,394 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2546, ppid=2544, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:52,402 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=408 2020-05-21 17:28:52,402 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=160.0K (1.2K/sec) 2020-05-21 17:28:52,402 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2544, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:52,402 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,407 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=462 2020-05-21 17:28:52,407 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=160.5K (1.2K/sec) 2020-05-21 17:28:52,407 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE has lock 2020-05-21 17:28:52,407 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished subprocedure pid=2546, resume processing parent pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,407 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished pid=2546, ppid=2544, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 351msec 2020-05-21 17:28:52,407 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2546 runningCount=16, activeCount=0 2020-05-21 17:28:52,407 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,407 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2544 runningCount=1, activeCount=1 2020-05-21 17:28:52,408 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE; cycles=0 2020-05-21 17:28:52,408 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2544, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,408 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2544, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:52,408 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2544, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,408 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2544, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE and Delete [2545, 2546] 2020-05-21 17:28:52,413 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=479 2020-05-21 17:28:52,413 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=161.0K (1.2K/sec) 2020-05-21 17:28:52,413 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2544, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE 2020-05-21 17:28:52,419 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=467 2020-05-21 17:28:52,419 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=161.4K (1.2K/sec) 2020-05-21 17:28:52,419 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2544, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE released the shared lock 2020-05-21 17:28:52,419 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2544, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=e855ef99e11ec78bead3c9e21e6da12f, REOPEN/MOVE in 1.2670sec 2020-05-21 17:28:52,419 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2544 runningCount=16, activeCount=0 2020-05-21 17:28:52,455 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=6fc9d3397f8a7edbd4dd190aaef975c3, source=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:52,455 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE, subproc=null 2020-05-21 17:28:52,556 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=161.4K (1.2K/sec) 2020-05-21 17:28:52,562 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:52,562 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=161.8K (1.2K/sec) 2020-05-21 17:28:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,563 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,563 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2547 runningCount=1, activeCount=1 2020-05-21 17:28:52,563 INFO [PEWorker-11] procedure.MasterProcedureScheduler: Took xlock for pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,563 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,575 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:52,575 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=162.2K (1.2K/sec) 2020-05-21 17:28:52,575 TRACE [PEWorker-11] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; cycles=0 2020-05-21 17:28:52,575 TRACE [PEWorker-11] procedure2.StateMachineProcedure: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,575 INFO [PEWorker-11] assignment.RegionStateStore: pid=2547 updating hbase:meta row=6fc9d3397f8a7edbd4dd190aaef975c3, regionState=CLOSING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:52,578 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 89, totalSize: 6 bytes 2020-05-21 17:28:52,578 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:52,578 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:52,578 INFO [PEWorker-11] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:52,578 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:52,578 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Stored pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE, children [pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:52,578 TRACE [PEWorker-11] wal.WALProcedureStore: Insert pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE, subproc=[pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:52,586 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:52,586 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=163.0K (1.2K/sec) 2020-05-21 17:28:52,586 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,586 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2547 runningCount=16, activeCount=0 2020-05-21 17:28:52,586 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,586 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2548 runningCount=1, activeCount=1 2020-05-21 17:28:52,587 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,596 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:52,596 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=163.4K (1.2K/sec) 2020-05-21 17:28:52,596 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=false, [] 2020-05-21 17:28:52,596 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,596 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:52,596 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,596 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,611 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:52,611 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=163.8K (1.2K/sec) 2020-05-21 17:28:52,611 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,620 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:52,620 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=164.2K (1.2K/sec) 2020-05-21 17:28:52,620 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2548 runningCount=16, activeCount=0 2020-05-21 17:28:52,747 TRACE [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:52,749 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 90, totalSize: 4 bytes 2020-05-21 17:28:52,749 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:52,770 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 516 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:52,771 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 516 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.53.53:44503 deadline: 1590082142770 executing as hbase 2020-05-21 17:28:52,771 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=CLOSING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3 regionState=CLOSED 2020-05-21 17:28:52,771 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,872 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=164.2K (1.2K/sec) 2020-05-21 17:28:52,877 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:52,877 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=164.6K (1.2K/sec) 2020-05-21 17:28:52,878 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=true, [pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:52,878 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,878 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,878 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:52,878 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2548, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,879 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2548 runningCount=1, activeCount=1 2020-05-21 17:28:52,879 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,879 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 516 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.53.53:44503 deadline: 1590082142770 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 108 totalTime: 109 2020-05-21 17:28:52,886 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:52,886 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=165.0K (1.2K/sec) 2020-05-21 17:28:52,887 INFO [PEWorker-1] assignment.RegionStateStore: pid=2547 updating hbase:meta row=6fc9d3397f8a7edbd4dd190aaef975c3, regionState=CLOSED 2020-05-21 17:28:52,889 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 91, totalSize: 6 bytes 2020-05-21 17:28:52,889 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:52,889 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2548, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:52,889 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2548, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,889 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,894 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:52,895 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=165.4K (1.2K/sec) 2020-05-21 17:28:52,895 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2548, ppid=2547, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:52,899 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:52,900 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=165.9K (1.2K/sec) 2020-05-21 17:28:52,900 TRACE [PEWorker-1] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:52,900 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,905 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:52,905 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=166.3K (1.2K/sec) 2020-05-21 17:28:52,905 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE has lock 2020-05-21 17:28:52,905 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished subprocedure pid=2548, resume processing parent pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,905 INFO [PEWorker-1] procedure2.ProcedureExecutor: Finished pid=2548, ppid=2547, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 311msec 2020-05-21 17:28:52,905 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2548 runningCount=16, activeCount=0 2020-05-21 17:28:52,905 DEBUG [PEWorker-1] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,905 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Execute pid=2547 runningCount=1, activeCount=1 2020-05-21 17:28:52,905 TRACE [PEWorker-1] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; cycles=0 2020-05-21 17:28:52,905 TRACE [PEWorker-1] procedure2.StateMachineProcedure: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,905 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2547 2020-05-21 17:28:52,906 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:52,906 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,906 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,911 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:52,911 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=166.6K (1.2K/sec) 2020-05-21 17:28:52,911 TRACE [PEWorker-1] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; cycles=0 2020-05-21 17:28:52,911 TRACE [PEWorker-1] procedure2.StateMachineProcedure: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,911 INFO [PEWorker-1] assignment.TransitRegionStateProcedure: Starting pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:52,911 TRACE [PEWorker-1] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=false, [] 2020-05-21 17:28:52,911 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Suspend pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,911 DEBUG [PEWorker-1] procedure2.RootProcedureState: Add procedure pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:52,911 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Store update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,911 TRACE [PEWorker-1] wal.WALProcedureStore: Update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:52,918 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:52,918 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=167.0K (1.2K/sec) 2020-05-21 17:28:52,918 TRACE [PEWorker-1] procedure2.ProcedureExecutor: Halt pid=2547 runningCount=16, activeCount=0 2020-05-21 17:28:53,061 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:53,062 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:53,062 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:53,062 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:53,062 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=true, [pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE] 2020-05-21 17:28:53,062 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,062 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE has lock 2020-05-21 17:28:53,062 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:53,062 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,062 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2547 runningCount=1, activeCount=1 2020-05-21 17:28:53,062 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; cycles=0 2020-05-21 17:28:53,062 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,063 INFO [PEWorker-12] assignment.RegionStateStore: pid=2547 updating hbase:meta row=6fc9d3397f8a7edbd4dd190aaef975c3, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:53,066 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 92, totalSize: 6 bytes 2020-05-21 17:28:53,066 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:53,066 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:53,067 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:53,067 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:53,067 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE, children [pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:53,067 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE, subproc=[pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:53,072 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:53,072 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=167.8K (1.2K/sec) 2020-05-21 17:28:53,073 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,073 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2547 runningCount=16, activeCount=0 2020-05-21 17:28:53,073 DEBUG [PEWorker-4] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,073 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Execute pid=2549 runningCount=1, activeCount=1 2020-05-21 17:28:53,073 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,080 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:53,080 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=168.1K (1.2K/sec) 2020-05-21 17:28:53,080 TRACE [PEWorker-4] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=false, [] 2020-05-21 17:28:53,080 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Suspend pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,080 DEBUG [PEWorker-4] procedure2.RootProcedureState: Add procedure pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:53,080 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Store update pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,080 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,085 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:53,085 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=168.5K (1.2K/sec) 2020-05-21 17:28:53,085 TRACE [PEWorker-4] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,090 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:53,090 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=168.8K (1.2K/sec) 2020-05-21 17:28:53,091 TRACE [PEWorker-4] procedure2.ProcedureExecutor: Halt pid=2549 runningCount=16, activeCount=0 2020-05-21 17:28:53,231 TRACE [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:53,237 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 93, totalSize: 4 bytes 2020-05-21 17:28:53,237 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:28:53,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726608213243481 ns. 2020-05-21 17:28:53,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 98982 ns. 2020-05-21 17:28:53,261 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 517 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 168 bytes 2020-05-21 17:28:53,261 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 517 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.53.53:44503 deadline: 1590082143261 executing as hbase 2020-05-21 17:28:53,262 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3 regionState=OPENED 2020-05-21 17:28:53,262 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,362 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=168.8K (1.2K/sec) 2020-05-21 17:28:53,368 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=356 2020-05-21 17:28:53,368 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=169.2K (1.2K/sec) 2020-05-21 17:28:53,368 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 6fc9d3397f8a7edbd4dd190aaef975c3, NAME => 'aeris_test_backup_restore,\x0D,1581768003353.6fc9d3397f8a7edbd4dd190aaef975c3.', STARTKEY => '\x0D', ENDKEY => '\x0E'}, ready=true, [pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:53,368 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,368 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,369 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2549, ppid=2547, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,369 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2549 runningCount=1, activeCount=1 2020-05-21 17:28:53,369 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 517 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 168 connection: 10.128.53.53:44503 deadline: 1590082143261 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 108 totalTime: 108 2020-05-21 17:28:53,369 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,374 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=359 2020-05-21 17:28:53,374 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=169.5K (1.3K/sec) 2020-05-21 17:28:53,374 INFO [PEWorker-14] assignment.RegionStateStore: pid=2547 updating hbase:meta row=6fc9d3397f8a7edbd4dd190aaef975c3, regionState=OPEN, openSeqNum=262, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:53,377 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 94, totalSize: 6 bytes 2020-05-21 17:28:53,377 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:53,377 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2549, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:53,377 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2549, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,377 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=361 2020-05-21 17:28:53,382 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=169.9K (1.3K/sec) 2020-05-21 17:28:53,382 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2549, ppid=2547, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:53,388 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=358 2020-05-21 17:28:53,388 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=170.2K (1.3K/sec) 2020-05-21 17:28:53,388 TRACE [PEWorker-14] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2547, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:53,388 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,393 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:53,393 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=170.6K (1.3K/sec) 2020-05-21 17:28:53,394 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE has lock 2020-05-21 17:28:53,394 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished subprocedure pid=2549, resume processing parent pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,394 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2549, ppid=2547, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 311msec 2020-05-21 17:28:53,394 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2549 runningCount=16, activeCount=0 2020-05-21 17:28:53,394 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,394 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Execute pid=2547 runningCount=1, activeCount=1 2020-05-21 17:28:53,394 TRACE [PEWorker-14] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE; cycles=0 2020-05-21 17:28:53,394 TRACE [PEWorker-14] procedure2.StateMachineProcedure: pid=2547, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,394 DEBUG [PEWorker-14] procedure2.RootProcedureState: Add procedure pid=2547, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:53,394 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Store update pid=2547, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,394 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2547, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE and Delete [2549, 2548] 2020-05-21 17:28:53,400 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:53,400 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=171.0K (1.3K/sec) 2020-05-21 17:28:53,400 TRACE [PEWorker-14] wal.WALProcedureStore: Update pid=2547, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE 2020-05-21 17:28:53,405 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=419 2020-05-21 17:28:53,405 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=171.5K (1.3K/sec) 2020-05-21 17:28:53,405 DEBUG [PEWorker-14] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) to run queue because: pid=2547, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE released the shared lock 2020-05-21 17:28:53,406 INFO [PEWorker-14] procedure2.ProcedureExecutor: Finished pid=2547, state=SUCCESS; TransitRegionStateProcedure table=aeris_test_backup_restore, region=6fc9d3397f8a7edbd4dd190aaef975c3, REOPEN/MOVE in 939msec 2020-05-21 17:28:53,406 TRACE [PEWorker-14] procedure2.ProcedureExecutor: Halt pid=2547 runningCount=16, activeCount=0 2020-05-21 17:28:53,463 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=707360501b0ab41efd74025af857bbb7, source=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:53,464 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE, subproc=null 2020-05-21 17:28:53,566 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=171.5K (1.3K/sec) 2020-05-21 17:28:53,573 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=470 2020-05-21 17:28:53,573 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=171.9K (1.3K/sec) 2020-05-21 17:28:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,574 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,574 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2550 runningCount=1, activeCount=1 2020-05-21 17:28:53,574 INFO [PEWorker-2] procedure.MasterProcedureScheduler: Took xlock for pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,574 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,582 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=473 2020-05-21 17:28:53,582 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=172.4K (1.3K/sec) 2020-05-21 17:28:53,582 TRACE [PEWorker-2] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; cycles=0 2020-05-21 17:28:53,582 TRACE [PEWorker-2] procedure2.StateMachineProcedure: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:53,582 INFO [PEWorker-2] assignment.RegionStateStore: pid=2550 updating hbase:meta row=707360501b0ab41efd74025af857bbb7, regionState=CLOSING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:53,585 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 95, totalSize: 6 bytes 2020-05-21 17:28:53,585 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:53,585 TRACE [PEWorker-2] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:53,585 INFO [PEWorker-2] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:53,585 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:53,585 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Stored pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE, children [pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:53,585 TRACE [PEWorker-2] wal.WALProcedureStore: Insert pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE, subproc=[pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:53,592 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=967 2020-05-21 17:28:53,592 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=173.3K (1.3K/sec) 2020-05-21 17:28:53,592 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,592 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2550 runningCount=16, activeCount=0 2020-05-21 17:28:53,592 DEBUG [PEWorker-2] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,592 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Execute pid=2551 runningCount=1, activeCount=1 2020-05-21 17:28:53,592 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,598 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=493 2020-05-21 17:28:53,598 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=173.8K (1.3K/sec) 2020-05-21 17:28:53,599 TRACE [PEWorker-2] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=false, [] 2020-05-21 17:28:53,599 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Suspend pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,599 DEBUG [PEWorker-2] procedure2.RootProcedureState: Add procedure pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:53,599 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Store update pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,599 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,604 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=495 2020-05-21 17:28:53,604 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=174.3K (1.3K/sec) 2020-05-21 17:28:53,604 TRACE [PEWorker-2] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:53,609 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=492 2020-05-21 17:28:53,609 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=174.8K (1.3K/sec) 2020-05-21 17:28:53,609 TRACE [PEWorker-2] procedure2.ProcedureExecutor: Halt pid=2551 runningCount=16, activeCount=0 2020-05-21 17:28:53,748 TRACE [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:53,751 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 96, totalSize: 4 bytes 2020-05-21 17:28:53,751 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 2ms 2020-05-21 17:28:54,073 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 523 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4071 bytes 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 523 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082144073 executing as hbase 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,074 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90599702, completeSequenceId=90604375 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:54,075 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 523 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082144073 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 633 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:28:54,170 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 518 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4274 bytes 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 518 service: RegionServerStatusService methodName: RegionServerReport size: 4.2 K connection: 10.128.53.53:44503 deadline: 1590082144171 executing as hbase 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,171 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 73993ad09b31c2a8063646a340a37327, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=-1, completeSequenceId=121135426 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121128147, completeSequenceId=121135426 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:54,172 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:54,173 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 518 service: RegionServerStatusService methodName: RegionServerReport size: 4.2 K connection: 10.128.53.53:44503 deadline: 1590082144171 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 666 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:54,187 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 519 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 245 bytes 2020-05-21 17:28:54,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 519 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 245 connection: 10.128.53.53:44503 deadline: 1590082144188 executing as hbase 2020-05-21 17:28:54,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=CLOSING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=707360501b0ab41efd74025af857bbb7 regionState=CLOSED 2020-05-21 17:28:54,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726609213253647 ns. 2020-05-21 17:28:54,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 122777 ns. 2020-05-21 17:28:54,289 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=174.8K (1.3K/sec) 2020-05-21 17:28:54,296 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=505 2020-05-21 17:28:54,296 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=175.3K (1.3K/sec) 2020-05-21 17:28:54,297 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=true, [pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:54,297 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,297 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,297 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=707360501b0ab41efd74025af857bbb7 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:54,297 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2551, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,298 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Execute pid=2551 runningCount=1, activeCount=1 2020-05-21 17:28:54,298 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 519 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 245 connection: 10.128.53.53:44503 deadline: 1590082144188 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 110 totalTime: 110 2020-05-21 17:28:54,298 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,304 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=508 2020-05-21 17:28:54,305 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=175.8K (1.3K/sec) 2020-05-21 17:28:54,305 INFO [PEWorker-9] assignment.RegionStateStore: pid=2550 updating hbase:meta row=707360501b0ab41efd74025af857bbb7, regionState=CLOSED 2020-05-21 17:28:54,307 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 97, totalSize: 6 bytes 2020-05-21 17:28:54,307 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:54,307 DEBUG [PEWorker-9] procedure2.RootProcedureState: Add procedure pid=2551, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:54,308 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Store update pid=2551, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,308 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,315 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=510 2020-05-21 17:28:54,315 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=176.3K (1.3K/sec) 2020-05-21 17:28:54,315 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2551, ppid=2550, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:54,320 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=507 2020-05-21 17:28:54,320 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=176.7K (1.3K/sec) 2020-05-21 17:28:54,320 TRACE [PEWorker-9] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:54,320 TRACE [PEWorker-9] wal.WALProcedureStore: Update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,327 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=481 2020-05-21 17:28:54,327 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=177.2K (1.3K/sec) 2020-05-21 17:28:54,327 DEBUG [PEWorker-9] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE has lock 2020-05-21 17:28:54,327 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished subprocedure pid=2551, resume processing parent pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,327 INFO [PEWorker-9] procedure2.ProcedureExecutor: Finished pid=2551, ppid=2550, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 722msec 2020-05-21 17:28:54,327 TRACE [PEWorker-9] procedure2.ProcedureExecutor: Halt pid=2551 runningCount=16, activeCount=0 2020-05-21 17:28:54,327 DEBUG [PEWorker-10] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,327 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Execute pid=2550 runningCount=1, activeCount=1 2020-05-21 17:28:54,327 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; cycles=0 2020-05-21 17:28:54,327 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,327 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2550 2020-05-21 17:28:54,327 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:54,327 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,327 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,334 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=485 2020-05-21 17:28:54,334 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=177.7K (1.3K/sec) 2020-05-21 17:28:54,334 TRACE [PEWorker-10] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; cycles=0 2020-05-21 17:28:54,334 TRACE [PEWorker-10] procedure2.StateMachineProcedure: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,334 INFO [PEWorker-10] assignment.TransitRegionStateProcedure: Starting pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:54,334 TRACE [PEWorker-10] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=false, [] 2020-05-21 17:28:54,334 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Suspend pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,334 DEBUG [PEWorker-10] procedure2.RootProcedureState: Add procedure pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:54,334 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Store update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,334 TRACE [PEWorker-10] wal.WALProcedureStore: Update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,340 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=489 2020-05-21 17:28:54,340 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=178.2K (1.3K/sec) 2020-05-21 17:28:54,340 TRACE [PEWorker-10] procedure2.ProcedureExecutor: Halt pid=2550 runningCount=16, activeCount=0 2020-05-21 17:28:54,484 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:54,485 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=2: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754] 2020-05-21 17:28:54,485 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:54,485 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 0 retained the pre-restart assignment. 1 regions were assigned to random hosts, since the old hosts for these regions are no longer present in the cluster. These hosts were: regionserver-2.hbase.hbase.svc.cluster.local 2020-05-21 17:28:54,485 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=true, [pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE] 2020-05-21 17:28:54,485 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,486 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE has lock 2020-05-21 17:28:54,486 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 1msec 2020-05-21 17:28:54,486 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,486 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2550 runningCount=1, activeCount=1 2020-05-21 17:28:54,486 TRACE [PEWorker-6] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; cycles=0 2020-05-21 17:28:54,486 TRACE [PEWorker-6] procedure2.StateMachineProcedure: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,486 INFO [PEWorker-6] assignment.RegionStateStore: pid=2550 updating hbase:meta row=707360501b0ab41efd74025af857bbb7, regionState=OPENING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:54,489 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 98, totalSize: 6 bytes 2020-05-21 17:28:54,489 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 3ms 2020-05-21 17:28:54,490 TRACE [PEWorker-6] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:54,491 INFO [PEWorker-6] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:54,491 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:54,491 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Stored pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE, children [pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:54,491 TRACE [PEWorker-6] wal.WALProcedureStore: Insert pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE, subproc=[pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:54,498 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=917 2020-05-21 17:28:54,498 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=179.1K (1.3K/sec) 2020-05-21 17:28:54,498 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,499 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2550 runningCount=16, activeCount=0 2020-05-21 17:28:54,499 DEBUG [PEWorker-6] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,499 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Execute pid=2552 runningCount=1, activeCount=1 2020-05-21 17:28:54,499 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,503 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=431 2020-05-21 17:28:54,503 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=179.5K (1.3K/sec) 2020-05-21 17:28:54,504 TRACE [PEWorker-6] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=false, [] 2020-05-21 17:28:54,504 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Suspend pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,504 DEBUG [PEWorker-6] procedure2.RootProcedureState: Add procedure pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:54,504 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Store update pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,504 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,509 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=433 2020-05-21 17:28:54,509 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=179.9K (1.3K/sec) 2020-05-21 17:28:54,509 TRACE [PEWorker-6] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,515 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:54,516 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=180.3K (1.3K/sec) 2020-05-21 17:28:54,516 TRACE [PEWorker-6] procedure2.ProcedureExecutor: Halt pid=2552 runningCount=16, activeCount=0 2020-05-21 17:28:54,575 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082134574, delay=0 2020-05-21 17:28:54,653 TRACE [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:54,660 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 99, totalSize: 4 bytes 2020-05-21 17:28:54,660 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 7ms 2020-05-21 17:28:54,802 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 520 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 250 bytes 2020-05-21 17:28:54,803 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 520 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 250 connection: 10.128.53.53:44503 deadline: 1590082144802 executing as hbase 2020-05-21 17:28:54,803 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=OPENING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_v2, region=707360501b0ab41efd74025af857bbb7 regionState=OPENED 2020-05-21 17:28:54,803 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,904 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=180.3K (1.3K/sec) 2020-05-21 17:28:54,910 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=437 2020-05-21 17:28:54,910 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=180.8K (1.3K/sec) 2020-05-21 17:28:54,910 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 707360501b0ab41efd74025af857bbb7, NAME => 'aeris_v2,\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7.', STARTKEY => '\x0C\xCD\x020\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter', ENDKEY => '\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642'}, ready=true, [pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:54,910 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,910 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,911 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 520 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 250 connection: 10.128.53.53:44503 deadline: 1590082144802 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 108 totalTime: 109 2020-05-21 17:28:54,911 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2552, ppid=2550, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,911 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Execute pid=2552 runningCount=1, activeCount=1 2020-05-21 17:28:54,911 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,916 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=440 2020-05-21 17:28:54,916 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=181.2K (1.3K/sec) 2020-05-21 17:28:54,917 INFO [PEWorker-15] assignment.RegionStateStore: pid=2550 updating hbase:meta row=707360501b0ab41efd74025af857bbb7, regionState=OPEN, openSeqNum=121135430, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:54,919 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 100, totalSize: 6 bytes 2020-05-21 17:28:54,919 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:54,920 DEBUG [PEWorker-15] procedure2.RootProcedureState: Add procedure pid=2552, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 7th rollback step 2020-05-21 17:28:54,920 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Store update pid=2552, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,920 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,925 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=442 2020-05-21 17:28:54,925 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=181.6K (1.3K/sec) 2020-05-21 17:28:54,925 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2552, ppid=2550, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:54,930 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=439 2020-05-21 17:28:54,930 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=182.0K (1.3K/sec) 2020-05-21 17:28:54,930 TRACE [PEWorker-15] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2550, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:54,931 TRACE [PEWorker-15] wal.WALProcedureStore: Update pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,936 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=493 2020-05-21 17:28:54,936 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=182.5K (1.3K/sec) 2020-05-21 17:28:54,936 DEBUG [PEWorker-15] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=1 size=1) to run queue because: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE has lock 2020-05-21 17:28:54,936 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished subprocedure pid=2552, resume processing parent pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,937 INFO [PEWorker-15] procedure2.ProcedureExecutor: Finished pid=2552, ppid=2550, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure in 431msec 2020-05-21 17:28:54,937 TRACE [PEWorker-15] procedure2.ProcedureExecutor: Halt pid=2552 runningCount=16, activeCount=0 2020-05-21 17:28:54,937 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_v2, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,937 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Execute pid=2550 runningCount=1, activeCount=1 2020-05-21 17:28:54,937 TRACE [PEWorker-13] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_OPENED pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE; cycles=0 2020-05-21 17:28:54,937 TRACE [PEWorker-13] procedure2.StateMachineProcedure: pid=2550, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,937 DEBUG [PEWorker-13] procedure2.RootProcedureState: Add procedure pid=2550, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE as the 8th rollback step 2020-05-21 17:28:54,937 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Store update pid=2550, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,937 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2550, state=SUCCESS, locked=true; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE and Delete [2552, 2551] 2020-05-21 17:28:54,944 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=510 2020-05-21 17:28:54,944 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=183.0K (1.3K/sec) 2020-05-21 17:28:54,945 TRACE [PEWorker-13] wal.WALProcedureStore: Update pid=2550, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE 2020-05-21 17:28:54,950 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=498 2020-05-21 17:28:54,950 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=183.5K (1.3K/sec) 2020-05-21 17:28:54,950 DEBUG [PEWorker-13] procedure.MasterProcedureScheduler: Add TableQueue(aeris_v2, xlock=false sharedLock=0 size=0) to run queue because: pid=2550, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE released the shared lock 2020-05-21 17:28:54,950 INFO [PEWorker-13] procedure2.ProcedureExecutor: Finished pid=2550, state=SUCCESS; TransitRegionStateProcedure table=aeris_v2, region=707360501b0ab41efd74025af857bbb7, REOPEN/MOVE in 1.4730sec 2020-05-21 17:28:54,951 TRACE [PEWorker-13] procedure2.ProcedureExecutor: Halt pid=2550 runningCount=16, activeCount=0 2020-05-21 17:28:54,975 INFO [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] master.HMaster: balance hri=73993ad09b31c2a8063646a340a37327, source=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, destination=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367 2020-05-21 17:28:54,975 TRACE [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] wal.WALProcedureStore: Insert pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE, subproc=null 2020-05-21 17:28:54,977 TRACE [RS-EventLoopGroup-1-2] ipc.NettyRpcServer: Connection /10.128.14.40:39393; # active connections=3 2020-05-21 17:28:55,005 INFO [RS-EventLoopGroup-1-2] hbase.Server: Connection from 10.128.14.40:39393, version=2.2.4, sasl=false, ugi=hbase (auth:SIMPLE), service=RegionServerStatusService 2020-05-21 17:28:55,005 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 0 method_name: "RegionServerStartup" request_param: true priority: 0 timeout: 10000 totalRequestSize: 95 bytes 2020-05-21 17:28:55,010 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 0 service: RegionServerStatusService methodName: RegionServerStartup size: 95 connection: 10.128.14.40:39393 deadline: 1590082145009 executing as hbase 2020-05-21 17:28:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] master.ServerManager: STARTUP: Server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 came back up, removed it from the dead servers list 2020-05-21 17:28:55,010 INFO [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] master.ServerManager: Registering regionserver=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:28:55,017 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 0 service: RegionServerStatusService methodName: RegionServerStartup size: 95 connection: 10.128.14.40:39393 deadline: 1590082145009 param: port: 16020 server_start_code: 1590082132059 server_current_time: 1590082134886 use_this_hostname_instead: "regionserver-2.hbase.hbase.svc.cluster.local" connection: 10.128.14.40:39393, response map_entries { name: "hbase.rootdir" value: "hdfs://aeris/hbase" } map_entries { name: "fs.defaultFS" value: "hdfs://aeris" } map_entries { name: "hbase.master.info.port" value: "16010" } map_entries { name: "hbase.regionserver.hostname.seen.by.master" value: "regionserver-2.hbase.hbase.svc.cluster.local" } queueTime: 1 processingTime: 5 totalTime: 6 2020-05-21 17:28:55,039 INFO [RegionServerTracker-0] master.RegionServerTracker: RegionServer ephemeral node created, adding [regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059] 2020-05-21 17:28:55,076 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 101msec, slotIndex=1 , totalSynced=183.5K (1.3K/sec) 2020-05-21 17:28:55,085 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=391 2020-05-21 17:28:55,085 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=183.9K (1.3K/sec) 2020-05-21 17:28:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure2.ProcedureExecutor: Stored pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=49,queue=4,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,087 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=0 size=0) from run queue because: queue is empty after polling out pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,087 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Execute pid=2553 runningCount=1, activeCount=1 2020-05-21 17:28:55,087 INFO [PEWorker-11] procedure.MasterProcedureScheduler: Took xlock for pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,087 TRACE [PEWorker-11] wal.WALProcedureStore: Update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,096 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=394 2020-05-21 17:28:55,096 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=184.3K (1.3K/sec) 2020-05-21 17:28:55,096 TRACE [PEWorker-11] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CLOSE pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE; cycles=0 2020-05-21 17:28:55,096 TRACE [PEWorker-11] procedure2.StateMachineProcedure: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,097 INFO [PEWorker-11] assignment.RegionStateStore: pid=2553 updating hbase:meta row=73993ad09b31c2a8063646a340a37327, regionState=CLOSING, regionLocation=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 2020-05-21 17:28:55,099 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 101, totalSize: 6 bytes 2020-05-21 17:28:55,099 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:55,099 TRACE [PEWorker-11] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:55,100 INFO [PEWorker-11] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure}] 2020-05-21 17:28:55,100 DEBUG [PEWorker-11] procedure2.RootProcedureState: Add procedure pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE as the 0th rollback step 2020-05-21 17:28:55,100 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Stored pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE, children [pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:55,100 TRACE [PEWorker-11] wal.WALProcedureStore: Insert pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE, subproc=[pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:55,106 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=809 2020-05-21 17:28:55,106 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=185.1K (1.3K/sec) 2020-05-21 17:28:55,106 DEBUG [PEWorker-11] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,106 TRACE [PEWorker-11] procedure2.ProcedureExecutor: Halt pid=2553 runningCount=16, activeCount=0 2020-05-21 17:28:55,106 DEBUG [PEWorker-8] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,106 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Execute pid=2554 runningCount=1, activeCount=1 2020-05-21 17:28:55,107 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,112 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=414 2020-05-21 17:28:55,112 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=185.5K (1.4K/sec) 2020-05-21 17:28:55,112 TRACE [PEWorker-8] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}, ready=false, [] 2020-05-21 17:28:55,113 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Suspend pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,113 DEBUG [PEWorker-8] procedure2.RootProcedureState: Add procedure pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 1th rollback step 2020-05-21 17:28:55,113 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Store update pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,113 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,117 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=416 2020-05-21 17:28:55,117 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=185.9K (1.4K/sec) 2020-05-21 17:28:55,117 TRACE [PEWorker-8] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,125 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=413 2020-05-21 17:28:55,125 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=186.3K (1.4K/sec) 2020-05-21 17:28:55,125 TRACE [PEWorker-8] procedure2.ProcedureExecutor: Halt pid=2554 runningCount=16, activeCount=0 2020-05-21 17:28:55,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726610213229742 ns. 2020-05-21 17:28:55,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 97740 ns. 2020-05-21 17:28:55,262 TRACE [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:55,264 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 102, totalSize: 4 bytes 2020-05-21 17:28:55,264 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:55,282 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 521 method_name: "ReportRegionStateTransition" request_param: true priority: 0 timeout: 10000 totalRequestSize: 165 bytes 2020-05-21 17:28:55,283 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 521 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.53.53:44503 deadline: 1590082145282 executing as hbase 2020-05-21 17:28:55,283 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: Update region transition serverName=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066 region=rit=CLOSING, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327 regionState=CLOSED 2020-05-21 17:28:55,283 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,384 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=186.3K (1.4K/sec) 2020-05-21 17:28:55,389 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=427 2020-05-21 17:28:55,389 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=186.7K (1.4K/sec) 2020-05-21 17:28:55,390 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}, ready=true, [pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure] 2020-05-21 17:28:55,390 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure2.AbstractProcedureScheduler: Wake pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,391 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,391 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.RegionStateNode: Tracking when we are set to null rit=CLOSED, location=regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327 java.lang.Throwable: TRACE at org.apache.hadoop.hbase.master.assignment.RegionStateNode.setRegionLocation(RegionStateNode.java:190) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.regionClosedWithoutPersistingToMeta(AssignmentManager.java:1713) at org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure.updateTransitionWithoutPersistingToMeta(CloseRegionProcedure.java:106) at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.reportTransition(RegionRemoteProcedureBase.java:220) at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.reportTransition(TransitRegionStateProcedure.java:395) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportTransition(AssignmentManager.java:986) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:958) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:874) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:919) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1612) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:15337) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:413) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:338) at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:318) 2020-05-21 17:28:55,391 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2554, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,391 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2554 runningCount=1, activeCount=1 2020-05-21 17:28:55,391 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,391 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 521 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 165 connection: 10.128.53.53:44503 deadline: 1590082145282 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$ReportRegionStateTransitionRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 108 totalTime: 109 2020-05-21 17:28:55,396 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=430 2020-05-21 17:28:55,396 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=187.1K (1.4K/sec) 2020-05-21 17:28:55,397 INFO [PEWorker-5] assignment.RegionStateStore: pid=2553 updating hbase:meta row=73993ad09b31c2a8063646a340a37327, regionState=CLOSED 2020-05-21 17:28:55,399 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 103, totalSize: 6 bytes 2020-05-21 17:28:55,399 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:55,399 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2554, ppid=2553, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure as the 2th rollback step 2020-05-21 17:28:55,399 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2554, ppid=2553, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,399 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=SUCCESS, locked=true; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,405 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=432 2020-05-21 17:28:55,405 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=187.5K (1.4K/sec) 2020-05-21 17:28:55,405 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2554, ppid=2553, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure 2020-05-21 17:28:55,412 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=429 2020-05-21 17:28:55,412 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=188.0K (1.4K/sec) 2020-05-21 17:28:55,412 TRACE [PEWorker-5] procedure2.Procedure: CHILD LATCH DECREMENT 0 java.lang.Throwable: pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.childrenCountDown(Procedure.java:874) at org.apache.hadoop.hbase.procedure2.Procedure.tryRunnable(Procedure.java:886) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.countDownChildren(ProcedureExecutor.java:1832) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1767) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:55,412 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,420 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=402 2020-05-21 17:28:55,420 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=188.4K (1.4K/sec) 2020-05-21 17:28:55,420 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE has lock 2020-05-21 17:28:55,420 INFO [PEWorker-5] procedure2.ProcedureExecutor: Finished subprocedure pid=2554, resume processing parent pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,420 INFO [PEWorker-5] procedure2.ProcedureExecutor: Finished pid=2554, ppid=2553, state=SUCCESS; org.apache.hadoop.hbase.master.assignment.CloseRegionProcedure in 300msec 2020-05-21 17:28:55,420 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2554 runningCount=16, activeCount=0 2020-05-21 17:28:55,420 DEBUG [PEWorker-5] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,420 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Execute pid=2553 runningCount=1, activeCount=1 2020-05-21 17:28:55,421 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_CONFIRM_CLOSED pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE; cycles=0 2020-05-21 17:28:55,421 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_CLOSED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,421 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Short-circuit to next step on pid=2553 2020-05-21 17:28:55,421 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE as the 3th rollback step 2020-05-21 17:28:55,421 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,421 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,426 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=406 2020-05-21 17:28:55,426 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=188.7K (1.4K/sec) 2020-05-21 17:28:55,426 TRACE [PEWorker-5] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE; cycles=0 2020-05-21 17:28:55,426 TRACE [PEWorker-5] procedure2.StateMachineProcedure: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,426 INFO [PEWorker-5] assignment.TransitRegionStateProcedure: Starting pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE; rit=CLOSED, location=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367; forceNewPlan=false, retain=false 2020-05-21 17:28:55,426 TRACE [PEWorker-5] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}, ready=false, [] 2020-05-21 17:28:55,426 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Suspend pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,426 DEBUG [PEWorker-5] procedure2.RootProcedureState: Add procedure pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE as the 4th rollback step 2020-05-21 17:28:55,426 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Store update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,426 TRACE [PEWorker-5] wal.WALProcedureStore: Update pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,432 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=410 2020-05-21 17:28:55,432 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=189.1K (1.4K/sec) 2020-05-21 17:28:55,432 TRACE [PEWorker-5] procedure2.ProcedureExecutor: Halt pid=2553 runningCount=16, activeCount=0 2020-05-21 17:28:55,576 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: PROCESS ASSIGN QUEUE regionCount=1 2020-05-21 17:28:55,577 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: Available servers count=3: [regionserver-0.hbase.hbase.svc.cluster.local,16020,1590081082066, regionserver-1.hbase.hbase.svc.cluster.local,16020,1590081081754, regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059] 2020-05-21 17:28:55,577 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: retain assign regions={{ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367} 2020-05-21 17:28:55,577 INFO [master/hbasemaster-0:16000] balancer.BaseLoadBalancer: Reassigned 1 regions. 1 retained the pre-restart assignment. 2020-05-21 17:28:55,577 TRACE [master/hbasemaster-0:16000] procedure2.ProcedureEvent: Unsuspend AssignmentProcedureEvent for {ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}, ready=true, [pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE] 2020-05-21 17:28:55,577 TRACE [master/hbasemaster-0:16000] procedure2.AbstractProcedureScheduler: Wake pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,577 DEBUG [master/hbasemaster-0:16000] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE has lock 2020-05-21 17:28:55,577 TRACE [master/hbasemaster-0:16000] assignment.AssignmentManager: ASSIGN ACCEPT 1 -> 0msec 2020-05-21 17:28:55,577 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,577 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2553 runningCount=1, activeCount=1 2020-05-21 17:28:55,578 TRACE [PEWorker-12] procedure2.StateMachineProcedure: REGION_STATE_TRANSITION_OPEN pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE; cycles=0 2020-05-21 17:28:55,578 TRACE [PEWorker-12] procedure2.StateMachineProcedure: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE 2020-05-21 17:28:55,578 INFO [PEWorker-12] assignment.RegionStateStore: pid=2553 updating hbase:meta row=73993ad09b31c2a8063646a340a37327, regionState=OPENING, regionLocation=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:28:55,580 TRACE [RS-EventLoopGroup-1-1] ipc.NettyRpcDuplexHandler: got response header call_id: 104, totalSize: 6 bytes 2020-05-21 17:28:55,580 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: Mutate, callTime: 2ms 2020-05-21 17:28:55,581 TRACE [PEWorker-12] procedure2.Procedure: CHILD LATCH INCREMENT SET 1 java.lang.Throwable: pid=2553, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE at org.apache.hadoop.hbase.procedure2.Procedure.setChildrenLatch(Procedure.java:852) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.initializeChildren(ProcedureExecutor.java:1798) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1696) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1409) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:78) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1979) 2020-05-21 17:28:55,581 INFO [PEWorker-12] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure}] 2020-05-21 17:28:55,581 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE as the 5th rollback step 2020-05-21 17:28:55,581 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Stored pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE, children [pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:55,581 TRACE [PEWorker-12] wal.WALProcedureStore: Insert pid=2553, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=aeris_test_backup_restore, region=73993ad09b31c2a8063646a340a37327, REOPEN/MOVE, subproc=[pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure] 2020-05-21 17:28:55,587 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=759 2020-05-21 17:28:55,587 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=189.9K (1.4K/sec) 2020-05-21 17:28:55,587 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Add TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=1) to run queue because: the exclusive lock is not held by anyone when adding pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,587 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2553 runningCount=16, activeCount=0 2020-05-21 17:28:55,587 DEBUG [PEWorker-12] procedure.MasterProcedureScheduler: Remove TableQueue(aeris_test_backup_restore, xlock=false sharedLock=1 size=0) from run queue because: queue is empty after polling out pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,587 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Execute pid=2555 runningCount=1, activeCount=1 2020-05-21 17:28:55,587 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2555, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=352 2020-05-21 17:28:55,593 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=190.2K (1.4K/sec) 2020-05-21 17:28:55,593 TRACE [PEWorker-12] procedure2.ProcedureEvent: Suspend AssignmentProcedureEvent for {ENCODED => 73993ad09b31c2a8063646a340a37327, NAME => 'aeris_test_backup_restore,\x06,1581768003353.73993ad09b31c2a8063646a340a37327.', STARTKEY => '\x06', ENDKEY => '\x07'}, ready=false, [] 2020-05-21 17:28:55,593 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Suspend pid=2555, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,593 DEBUG [PEWorker-12] procedure2.RootProcedureState: Add procedure pid=2555, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure as the 6th rollback step 2020-05-21 17:28:55,593 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Store update pid=2555, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,593 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2555, ppid=2553, state=RUNNABLE, locked=true; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,598 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=354 2020-05-21 17:28:55,598 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=190.6K (1.4K/sec) 2020-05-21 17:28:55,598 TRACE [PEWorker-12] wal.WALProcedureStore: Update pid=2555, ppid=2553, state=RUNNABLE; org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure 2020-05-21 17:28:55,603 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/1, flushed=351 2020-05-21 17:28:55,603 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=190.9K (1.4K/sec) 2020-05-21 17:28:55,603 TRACE [PEWorker-12] procedure2.ProcedureExecutor: Halt pid=2555 runningCount=16, activeCount=0 2020-05-21 17:28:55,744 TRACE [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:55,745 DEBUG [RSProcedureDispatcher-pool3-t31] master.ServerManager: New admin connection to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:28:55,745 TRACE [RSProcedureDispatcher-pool3-t31] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:55,755 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 10ms 2020-05-21 17:28:55,755 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:55,755 DEBUG [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=0 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:28:55,757 WARN [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=0, retrying... 2020-05-21 17:28:55,858 TRACE [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:55,859 DEBUG [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:55,859 TRACE [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:55,859 DEBUG [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=1 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:55,860 WARN [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=1, retrying... 2020-05-21 17:28:55,960 TRACE [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:55,961 DEBUG [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:55,961 TRACE [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:55,961 DEBUG [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=2 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:55,962 WARN [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=2, retrying... 2020-05-21 17:28:56,062 TRACE [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,063 DEBUG [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,063 TRACE [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,063 DEBUG [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=3 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,063 WARN [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=3, retrying... 2020-05-21 17:28:56,164 TRACE [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,164 DEBUG [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,165 TRACE [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:56,165 DEBUG [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=4 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,165 WARN [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=4, retrying... 2020-05-21 17:28:56,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726611213264835 ns. 2020-05-21 17:28:56,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 109316 ns. 2020-05-21 17:28:56,266 TRACE [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,266 DEBUG [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,266 TRACE [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,266 DEBUG [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=5 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,267 WARN [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=5, retrying... 2020-05-21 17:28:56,367 TRACE [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,368 DEBUG [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,368 TRACE [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,368 DEBUG [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=6 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,368 WARN [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=6, retrying... 2020-05-21 17:28:56,469 TRACE [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,469 DEBUG [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,470 TRACE [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:56,470 DEBUG [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=7 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,481 WARN [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=7, retrying... 2020-05-21 17:28:56,582 TRACE [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,582 DEBUG [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,582 TRACE [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,582 DEBUG [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=8 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,582 WARN [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=8, retrying... 2020-05-21 17:28:56,683 TRACE [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,684 DEBUG [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,684 TRACE [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,684 DEBUG [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=9 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,684 WARN [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=9, retrying... 2020-05-21 17:28:56,785 TRACE [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,785 DEBUG [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,785 TRACE [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,785 DEBUG [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=10 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,787 WARN [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=10, retrying... 2020-05-21 17:28:56,888 TRACE [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,889 DEBUG [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,889 TRACE [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,889 DEBUG [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=11 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,890 WARN [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=11, retrying... 2020-05-21 17:28:56,991 TRACE [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:56,991 DEBUG [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:56,991 TRACE [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:56,991 DEBUG [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=12 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:56,992 WARN [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=12, retrying... 2020-05-21 17:28:57,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:28:57,077 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 524 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4074 bytes 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 524 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082147078 executing as hbase 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,078 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:28:57,079 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 524 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082147078 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 633 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:28:57,092 TRACE [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,092 DEBUG [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,092 TRACE [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,093 DEBUG [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=13 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,093 WARN [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=13, retrying... 2020-05-21 17:28:57,175 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 522 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:28:57,176 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 522 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082147175 executing as hbase 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:28:57,177 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135426, completeSequenceId=121135431 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:28:57,178 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:28:57,179 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 522 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082147175 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 666 } connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 3 totalTime: 4 2020-05-21 17:28:57,194 TRACE [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,194 DEBUG [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,194 TRACE [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,194 DEBUG [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=14 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,195 WARN [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=14, retrying... 2020-05-21 17:28:57,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726612213253937 ns. 2020-05-21 17:28:57,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 97684 ns. 2020-05-21 17:28:57,295 TRACE [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,296 DEBUG [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,296 TRACE [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:57,296 DEBUG [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=15 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,296 WARN [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=15, retrying... 2020-05-21 17:28:57,397 TRACE [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,397 DEBUG [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,397 TRACE [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,398 DEBUG [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=16 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,399 WARN [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=16, retrying... 2020-05-21 17:28:57,499 TRACE [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,499 DEBUG [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,499 TRACE [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,500 DEBUG [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=17 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,500 WARN [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=17, retrying... 2020-05-21 17:28:57,601 TRACE [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,601 DEBUG [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,601 TRACE [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,601 DEBUG [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=18 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,602 WARN [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=18, retrying... 2020-05-21 17:28:57,702 TRACE [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,703 DEBUG [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,703 TRACE [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:57,703 DEBUG [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=19 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,703 WARN [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=19, retrying... 2020-05-21 17:28:57,804 TRACE [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,804 TRACE [RSProcedureDispatcher-pool3-t51] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:57,808 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:28:57,808 DEBUG [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=20 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:28:57,808 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:57,810 WARN [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=20, retrying... 2020-05-21 17:28:57,911 TRACE [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:57,911 DEBUG [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:57,911 TRACE [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:57,911 DEBUG [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=21 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:57,916 WARN [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=21, retrying... 2020-05-21 17:28:58,020 TRACE [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,020 DEBUG [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,020 TRACE [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,020 DEBUG [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=22 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,021 WARN [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=22, retrying... 2020-05-21 17:28:58,121 TRACE [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,122 DEBUG [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,122 TRACE [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,122 DEBUG [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=23 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,123 WARN [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=23, retrying... 2020-05-21 17:28:58,224 TRACE [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,224 DEBUG [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,224 TRACE [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,224 DEBUG [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=24 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,225 WARN [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=24, retrying... 2020-05-21 17:28:58,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726613213245597 ns. 2020-05-21 17:28:58,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 132378 ns. 2020-05-21 17:28:58,325 TRACE [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,326 DEBUG [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,326 TRACE [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:58,326 DEBUG [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=25 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,326 WARN [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=25, retrying... 2020-05-21 17:28:58,427 TRACE [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,427 DEBUG [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,427 TRACE [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,427 DEBUG [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=26 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,428 WARN [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=26, retrying... 2020-05-21 17:28:58,528 TRACE [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,529 DEBUG [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,529 TRACE [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:58,529 DEBUG [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=27 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,529 WARN [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=27, retrying... 2020-05-21 17:28:58,630 TRACE [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,630 DEBUG [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,630 TRACE [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,630 DEBUG [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=28 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,630 WARN [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=28, retrying... 2020-05-21 17:28:58,705 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 1 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 129 bytes 2020-05-21 17:28:58,705 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 1 service: RegionServerStatusService methodName: RegionServerReport size: 129 connection: 10.128.14.40:39393 deadline: 1590082148705 executing as hbase 2020-05-21 17:28:58,705 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:28:58,705 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:28:58,706 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 1 service: RegionServerStatusService methodName: RegionServerReport size: 129 connection: 10.128.14.40:39393 deadline: 1590082148705 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:28:58,731 TRACE [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,732 DEBUG [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,733 TRACE [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:58,733 DEBUG [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=29 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,733 WARN [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=29, retrying... 2020-05-21 17:28:58,834 TRACE [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,834 DEBUG [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,834 TRACE [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:58,834 DEBUG [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=30 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,835 WARN [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=30, retrying... 2020-05-21 17:28:58,936 TRACE [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:58,936 DEBUG [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:58,937 TRACE [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:28:58,937 DEBUG [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=31 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:58,937 WARN [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=31, retrying... 2020-05-21 17:28:59,038 TRACE [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,038 DEBUG [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,038 TRACE [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,039 DEBUG [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=32 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,039 WARN [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=32, retrying... 2020-05-21 17:28:59,140 TRACE [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,140 DEBUG [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,140 TRACE [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,140 DEBUG [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=33 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,140 WARN [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=33, retrying... 2020-05-21 17:28:59,241 TRACE [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,241 DEBUG [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,241 TRACE [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,241 DEBUG [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=34 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,242 WARN [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=34, retrying... 2020-05-21 17:28:59,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726614213271712 ns. 2020-05-21 17:28:59,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 103543 ns. 2020-05-21 17:28:59,342 TRACE [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,343 DEBUG [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,343 TRACE [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,343 DEBUG [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=35 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,344 WARN [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=35, retrying... 2020-05-21 17:28:59,445 TRACE [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,445 DEBUG [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,445 TRACE [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,445 DEBUG [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=36 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,446 WARN [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=36, retrying... 2020-05-21 17:28:59,546 TRACE [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,547 DEBUG [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,547 TRACE [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,547 DEBUG [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=37 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,547 WARN [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=37, retrying... 2020-05-21 17:28:59,575 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082139575, delay=0 2020-05-21 17:28:59,648 TRACE [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,648 DEBUG [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,648 TRACE [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,648 DEBUG [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=38 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,649 WARN [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=38, retrying... 2020-05-21 17:28:59,749 TRACE [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,750 DEBUG [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,750 TRACE [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,750 DEBUG [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=39 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,750 WARN [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=39, retrying... 2020-05-21 17:28:59,851 TRACE [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,852 TRACE [RSProcedureDispatcher-pool3-t71] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:59,857 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:28:59,857 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:28:59,857 DEBUG [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=40 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:28:59,859 WARN [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=40, retrying... 2020-05-21 17:28:59,960 TRACE [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:28:59,960 DEBUG [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:28:59,960 TRACE [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:28:59,960 DEBUG [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=41 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:28:59,961 WARN [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=41, retrying... 2020-05-21 17:29:00,061 TRACE [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,062 DEBUG [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,062 TRACE [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,062 DEBUG [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=42 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,063 WARN [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=42, retrying... 2020-05-21 17:29:00,082 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 525 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4074 bytes 2020-05-21 17:29:00,082 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 525 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082150082 executing as hbase 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:00,083 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 525 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082150082 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 797 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:00,164 TRACE [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,164 DEBUG [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,164 TRACE [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,164 DEBUG [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=43 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,164 WARN [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=43, retrying... 2020-05-21 17:29:00,182 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 523 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4139 bytes 2020-05-21 17:29:00,182 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 523 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082150182 executing as hbase 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:00,183 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 523 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082150182 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 751 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:00,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726615213228702 ns. 2020-05-21 17:29:00,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 122772 ns. 2020-05-21 17:29:00,265 TRACE [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,265 DEBUG [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,265 TRACE [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,265 DEBUG [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=44 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,266 WARN [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=44, retrying... 2020-05-21 17:29:00,366 TRACE [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,367 DEBUG [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,367 TRACE [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,367 DEBUG [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=45 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,367 WARN [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=45, retrying... 2020-05-21 17:29:00,468 TRACE [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,468 DEBUG [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,468 TRACE [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,469 DEBUG [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=46 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,469 WARN [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=46, retrying... 2020-05-21 17:29:00,570 TRACE [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,570 DEBUG [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,570 TRACE [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,570 DEBUG [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=47 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,570 WARN [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=47, retrying... 2020-05-21 17:29:00,670 TRACE [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,670 DEBUG [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,670 TRACE [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,671 DEBUG [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=48 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,671 WARN [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=48, retrying... 2020-05-21 17:29:00,772 TRACE [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,772 DEBUG [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,772 TRACE [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,772 DEBUG [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=49 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,773 WARN [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=49, retrying... 2020-05-21 17:29:00,873 TRACE [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,874 DEBUG [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,874 TRACE [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,874 DEBUG [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=50 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,874 WARN [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=50, retrying... 2020-05-21 17:29:00,975 TRACE [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:00,975 DEBUG [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:00,975 TRACE [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:00,975 DEBUG [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=51 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:00,976 WARN [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=51, retrying... 2020-05-21 17:29:01,076 TRACE [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,077 DEBUG [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,077 TRACE [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,077 DEBUG [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=52 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,077 WARN [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=52, retrying... 2020-05-21 17:29:01,178 TRACE [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,178 DEBUG [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,178 TRACE [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,179 DEBUG [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=53 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,179 WARN [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=53, retrying... 2020-05-21 17:29:01,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726616213236492 ns. 2020-05-21 17:29:01,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 128189 ns. 2020-05-21 17:29:01,280 TRACE [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,280 DEBUG [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,280 TRACE [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,280 DEBUG [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=54 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,280 WARN [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=54, retrying... 2020-05-21 17:29:01,381 TRACE [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,383 DEBUG [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,383 TRACE [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:01,383 DEBUG [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=55 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,383 WARN [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=55, retrying... 2020-05-21 17:29:01,484 TRACE [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,484 DEBUG [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,484 TRACE [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,484 DEBUG [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=56 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,485 WARN [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=56, retrying... 2020-05-21 17:29:01,585 TRACE [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,586 DEBUG [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,586 TRACE [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,586 DEBUG [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=57 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,586 WARN [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=57, retrying... 2020-05-21 17:29:01,687 TRACE [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,687 DEBUG [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,687 TRACE [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,687 DEBUG [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=58 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,688 WARN [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=58, retrying... 2020-05-21 17:29:01,710 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 2 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:01,711 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 2 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082151711 executing as hbase 2020-05-21 17:29:01,711 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:01,712 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:01,712 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 2 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082151711 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:01,788 TRACE [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,789 DEBUG [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,789 TRACE [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,789 DEBUG [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=59 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,789 WARN [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=59, retrying... 2020-05-21 17:29:01,890 TRACE [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,890 TRACE [RSProcedureDispatcher-pool3-t91] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:01,894 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:01,894 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:01,894 DEBUG [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=60 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:01,896 WARN [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=60, retrying... 2020-05-21 17:29:01,997 TRACE [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:01,997 DEBUG [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:01,997 TRACE [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:01,997 DEBUG [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=61 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:01,998 WARN [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=61, retrying... 2020-05-21 17:29:02,098 TRACE [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,099 DEBUG [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,099 TRACE [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,099 DEBUG [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=62 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,101 WARN [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=62, retrying... 2020-05-21 17:29:02,202 TRACE [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,202 DEBUG [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,202 TRACE [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,202 DEBUG [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=63 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,202 WARN [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=63, retrying... 2020-05-21 17:29:02,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726617213251990 ns. 2020-05-21 17:29:02,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 100453 ns. 2020-05-21 17:29:02,302 TRACE [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,302 DEBUG [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,303 TRACE [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:02,303 DEBUG [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=64 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,303 WARN [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=64, retrying... 2020-05-21 17:29:02,404 TRACE [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,404 DEBUG [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,404 TRACE [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,404 DEBUG [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=65 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,404 WARN [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=65, retrying... 2020-05-21 17:29:02,505 TRACE [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,505 DEBUG [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,505 TRACE [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,506 DEBUG [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=66 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,506 WARN [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=66, retrying... 2020-05-21 17:29:02,607 TRACE [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,607 DEBUG [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,607 TRACE [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,607 DEBUG [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=67 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,607 WARN [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=67, retrying... 2020-05-21 17:29:02,708 TRACE [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,708 DEBUG [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,709 TRACE [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,709 DEBUG [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=68 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,709 WARN [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=68, retrying... 2020-05-21 17:29:02,810 TRACE [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,810 DEBUG [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,810 TRACE [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,810 DEBUG [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=69 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,811 WARN [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=69, retrying... 2020-05-21 17:29:02,913 TRACE [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:02,913 DEBUG [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:02,913 TRACE [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:02,914 DEBUG [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=70 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:02,914 WARN [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=70, retrying... 2020-05-21 17:29:03,015 TRACE [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,015 DEBUG [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,015 TRACE [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,015 DEBUG [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=71 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,016 WARN [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=71, retrying... 2020-05-21 17:29:03,086 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 526 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4075 bytes 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 526 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082153087 executing as hbase 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,087 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:03,088 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 526 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082153087 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 835 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:03,116 TRACE [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,117 DEBUG [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,117 TRACE [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,117 DEBUG [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=72 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,117 WARN [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=72, retrying... 2020-05-21 17:29:03,186 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 524 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4139 bytes 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 524 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082153186 executing as hbase 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:03,187 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:03,188 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 524 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082153186 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 765 } connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:29:03,218 TRACE [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,218 DEBUG [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,218 TRACE [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,218 DEBUG [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=73 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,219 WARN [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=73, retrying... 2020-05-21 17:29:03,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726618213274906 ns. 2020-05-21 17:29:03,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 137104 ns. 2020-05-21 17:29:03,320 TRACE [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,320 DEBUG [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,320 TRACE [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,329 DEBUG [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=74 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,329 WARN [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=74, retrying... 2020-05-21 17:29:03,430 TRACE [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,430 DEBUG [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,430 TRACE [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,431 DEBUG [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=75 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,431 WARN [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=75, retrying... 2020-05-21 17:29:03,532 TRACE [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,532 DEBUG [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,532 TRACE [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,532 DEBUG [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=76 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,532 WARN [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=76, retrying... 2020-05-21 17:29:03,633 TRACE [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,633 DEBUG [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,633 TRACE [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,634 DEBUG [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=77 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,634 WARN [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=77, retrying... 2020-05-21 17:29:03,735 TRACE [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,735 DEBUG [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,735 TRACE [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,735 DEBUG [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=78 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,735 WARN [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=78, retrying... 2020-05-21 17:29:03,836 TRACE [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,836 DEBUG [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:03,837 TRACE [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:03,837 DEBUG [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=79 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:03,837 WARN [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=79, retrying... 2020-05-21 17:29:03,938 TRACE [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:03,938 TRACE [RSProcedureDispatcher-pool3-t111] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:03,942 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:03,943 DEBUG [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=80 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:03,943 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:03,945 WARN [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=80, retrying... 2020-05-21 17:29:04,045 TRACE [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,046 DEBUG [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,046 TRACE [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,046 DEBUG [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=81 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,047 WARN [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=81, retrying... 2020-05-21 17:29:04,148 TRACE [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,148 DEBUG [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,148 TRACE [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,148 DEBUG [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=82 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,149 WARN [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=82, retrying... 2020-05-21 17:29:04,249 TRACE [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,249 DEBUG [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,250 TRACE [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:04,250 DEBUG [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=83 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,250 WARN [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=83, retrying... 2020-05-21 17:29:04,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726619213235878 ns. 2020-05-21 17:29:04,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 115436 ns. 2020-05-21 17:29:04,351 TRACE [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,351 DEBUG [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,351 TRACE [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,351 DEBUG [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=84 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,351 WARN [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=84, retrying... 2020-05-21 17:29:04,451 TRACE [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,451 DEBUG [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,451 TRACE [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,451 DEBUG [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=85 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,452 WARN [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=85, retrying... 2020-05-21 17:29:04,553 TRACE [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,554 DEBUG [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,554 TRACE [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:04,554 DEBUG [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=86 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,554 WARN [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=86, retrying... 2020-05-21 17:29:04,575 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082144575, delay=0 2020-05-21 17:29:04,655 TRACE [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,655 DEBUG [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,655 TRACE [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,655 DEBUG [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=87 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,656 WARN [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=87, retrying... 2020-05-21 17:29:04,716 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 3 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:04,716 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 3 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082154716 executing as hbase 2020-05-21 17:29:04,716 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:04,716 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:04,716 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 3 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082154716 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:04,756 TRACE [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,757 DEBUG [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,757 TRACE [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:04,757 DEBUG [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=88 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,757 WARN [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=88, retrying... 2020-05-21 17:29:04,858 TRACE [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,858 DEBUG [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,858 TRACE [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,858 DEBUG [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=89 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,858 WARN [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=89, retrying... 2020-05-21 17:29:04,959 TRACE [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:04,959 DEBUG [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:04,959 TRACE [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:04,959 DEBUG [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=90 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:04,960 WARN [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=90, retrying... 2020-05-21 17:29:05,060 TRACE [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,061 DEBUG [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,061 TRACE [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,061 DEBUG [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=91 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,061 WARN [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=91, retrying... 2020-05-21 17:29:05,162 TRACE [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,162 DEBUG [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,162 TRACE [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,162 DEBUG [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=92 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,163 WARN [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=92, retrying... 2020-05-21 17:29:05,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726620213268569 ns. 2020-05-21 17:29:05,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 95815 ns. 2020-05-21 17:29:05,263 TRACE [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,264 DEBUG [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,264 TRACE [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,264 DEBUG [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=93 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,264 WARN [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=93, retrying... 2020-05-21 17:29:05,365 TRACE [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,365 DEBUG [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,365 TRACE [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,365 DEBUG [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=94 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,366 WARN [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=94, retrying... 2020-05-21 17:29:05,466 TRACE [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,467 DEBUG [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,467 TRACE [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,467 DEBUG [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=95 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,467 WARN [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=95, retrying... 2020-05-21 17:29:05,568 TRACE [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,568 DEBUG [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,568 TRACE [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,568 DEBUG [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=96 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,569 WARN [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=96, retrying... 2020-05-21 17:29:05,669 TRACE [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,670 DEBUG [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,671 TRACE [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:05,671 DEBUG [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=97 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,671 WARN [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=97, retrying... 2020-05-21 17:29:05,772 TRACE [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,772 DEBUG [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,772 TRACE [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,772 DEBUG [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=98 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,773 WARN [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=98, retrying... 2020-05-21 17:29:05,873 TRACE [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,873 DEBUG [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:05,874 TRACE [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:05,874 DEBUG [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=99 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:05,874 WARN [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=99, retrying... 2020-05-21 17:29:05,974 TRACE [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:05,975 TRACE [RSProcedureDispatcher-pool3-t3] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:05,980 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:05,980 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:05,980 DEBUG [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=100 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:05,981 WARN [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=100, retrying... 2020-05-21 17:29:06,082 TRACE [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,082 DEBUG [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,082 TRACE [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,082 DEBUG [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=101 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,083 WARN [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=101, retrying... 2020-05-21 17:29:06,091 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 527 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4075 bytes 2020-05-21 17:29:06,091 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 527 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082156091 executing as hbase 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:06,092 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 527 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082156091 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 835 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:06,183 TRACE [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,184 DEBUG [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,184 TRACE [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,184 DEBUG [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=102 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,185 WARN [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=102, retrying... 2020-05-21 17:29:06,191 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 525 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4139 bytes 2020-05-21 17:29:06,191 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 525 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082156191 executing as hbase 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:06,192 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 525 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082156191 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 765 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:06,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726621213236216 ns. 2020-05-21 17:29:06,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 125304 ns. 2020-05-21 17:29:06,285 TRACE [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,285 DEBUG [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,285 TRACE [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,286 DEBUG [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=103 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,286 WARN [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=103, retrying... 2020-05-21 17:29:06,386 TRACE [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,387 DEBUG [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,387 TRACE [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,388 DEBUG [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=104 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,388 WARN [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=104, retrying... 2020-05-21 17:29:06,488 TRACE [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,489 DEBUG [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,489 TRACE [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,489 DEBUG [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=105 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,489 WARN [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=105, retrying... 2020-05-21 17:29:06,589 TRACE [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,590 DEBUG [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,590 TRACE [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,590 DEBUG [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=106 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,590 WARN [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=106, retrying... 2020-05-21 17:29:06,691 TRACE [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,691 DEBUG [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,691 TRACE [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,691 DEBUG [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=107 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,691 WARN [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=107, retrying... 2020-05-21 17:29:06,792 TRACE [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,792 DEBUG [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,793 TRACE [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:06,793 DEBUG [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=108 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,793 WARN [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=108, retrying... 2020-05-21 17:29:06,894 TRACE [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,894 DEBUG [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,894 TRACE [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,894 DEBUG [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=109 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,894 WARN [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=109, retrying... 2020-05-21 17:29:06,995 TRACE [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:06,995 DEBUG [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:06,995 TRACE [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:06,995 DEBUG [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=110 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:06,996 WARN [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=110, retrying... 2020-05-21 17:29:07,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:29:07,096 TRACE [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,096 DEBUG [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,096 TRACE [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,096 DEBUG [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=111 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,097 WARN [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=111, retrying... 2020-05-21 17:29:07,198 TRACE [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,198 DEBUG [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,199 TRACE [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:07,199 DEBUG [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=112 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,200 WARN [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=112, retrying... 2020-05-21 17:29:07,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726622213268437 ns. 2020-05-21 17:29:07,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 94892 ns. 2020-05-21 17:29:07,300 TRACE [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,301 DEBUG [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,301 TRACE [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:07,301 DEBUG [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=113 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,301 WARN [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=113, retrying... 2020-05-21 17:29:07,402 TRACE [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,403 DEBUG [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,403 TRACE [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,403 DEBUG [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=114 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,405 WARN [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=114, retrying... 2020-05-21 17:29:07,505 TRACE [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,505 DEBUG [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,505 TRACE [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,505 DEBUG [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=115 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,506 WARN [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=115, retrying... 2020-05-21 17:29:07,606 TRACE [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,606 DEBUG [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,606 TRACE [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,607 DEBUG [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=116 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,607 WARN [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=116, retrying... 2020-05-21 17:29:07,707 TRACE [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,707 DEBUG [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,707 TRACE [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,708 DEBUG [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=117 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,708 WARN [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=117, retrying... 2020-05-21 17:29:07,720 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 4 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:07,721 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 4 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082157720 executing as hbase 2020-05-21 17:29:07,721 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:07,721 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:07,721 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 4 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082157720 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:07,808 TRACE [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,808 DEBUG [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,809 TRACE [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:07,809 DEBUG [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=118 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,809 WARN [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=118, retrying... 2020-05-21 17:29:07,909 TRACE [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:07,910 DEBUG [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:07,910 TRACE [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:07,910 DEBUG [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=119 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:07,911 WARN [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=119, retrying... 2020-05-21 17:29:08,011 TRACE [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,011 TRACE [RSProcedureDispatcher-pool3-t23] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:08,016 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:08,016 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:08,016 DEBUG [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=120 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:08,018 WARN [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=120, retrying... 2020-05-21 17:29:08,118 TRACE [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,119 DEBUG [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,119 TRACE [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,119 DEBUG [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=121 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,119 WARN [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=121, retrying... 2020-05-21 17:29:08,219 TRACE [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,220 DEBUG [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,220 TRACE [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,220 DEBUG [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=122 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,221 WARN [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=122, retrying... 2020-05-21 17:29:08,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726623213220592 ns. 2020-05-21 17:29:08,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 96981 ns. 2020-05-21 17:29:08,321 TRACE [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,321 DEBUG [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,321 TRACE [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,321 DEBUG [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=123 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,322 WARN [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=123, retrying... 2020-05-21 17:29:08,422 TRACE [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,422 DEBUG [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,422 TRACE [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,423 DEBUG [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=124 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,423 WARN [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=124, retrying... 2020-05-21 17:29:08,523 TRACE [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,524 DEBUG [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,524 TRACE [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,524 DEBUG [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=125 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,524 WARN [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=125, retrying... 2020-05-21 17:29:08,624 TRACE [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,624 DEBUG [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,624 TRACE [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,624 DEBUG [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=126 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,625 WARN [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=126, retrying... 2020-05-21 17:29:08,725 TRACE [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,726 DEBUG [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,726 TRACE [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,726 DEBUG [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=127 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,739 WARN [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=127, retrying... 2020-05-21 17:29:08,839 TRACE [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,839 DEBUG [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,839 TRACE [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,839 DEBUG [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=128 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,840 WARN [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=128, retrying... 2020-05-21 17:29:08,940 TRACE [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:08,940 DEBUG [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:08,940 TRACE [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:08,941 DEBUG [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=129 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:08,941 WARN [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=129, retrying... 2020-05-21 17:29:09,041 TRACE [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,042 DEBUG [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,043 TRACE [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,043 DEBUG [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=130 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,043 WARN [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=130, retrying... 2020-05-21 17:29:09,095 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 528 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 528 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082159095 executing as hbase 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:09,096 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:09,097 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 528 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082159095 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 888 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:29:09,143 TRACE [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,144 DEBUG [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,144 TRACE [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,144 DEBUG [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=131 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,146 WARN [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=131, retrying... 2020-05-21 17:29:09,195 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 526 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:29:09,195 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 526 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082159195 executing as hbase 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:09,196 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 526 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082159195 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 892 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:09,246 TRACE [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,246 DEBUG [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,246 TRACE [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,246 DEBUG [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=132 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,247 WARN [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=132, retrying... 2020-05-21 17:29:09,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726624213271258 ns. 2020-05-21 17:29:09,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 95665 ns. 2020-05-21 17:29:09,347 TRACE [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,347 DEBUG [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,347 TRACE [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,348 DEBUG [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=133 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,348 WARN [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=133, retrying... 2020-05-21 17:29:09,448 TRACE [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,449 DEBUG [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,449 TRACE [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,449 DEBUG [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=134 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,449 WARN [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=134, retrying... 2020-05-21 17:29:09,549 TRACE [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,549 DEBUG [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,549 TRACE [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,549 DEBUG [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=135 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,550 WARN [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=135, retrying... 2020-05-21 17:29:09,576 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082149575, delay=0 2020-05-21 17:29:09,650 TRACE [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,650 DEBUG [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,650 TRACE [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,651 DEBUG [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=136 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,651 WARN [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=136, retrying... 2020-05-21 17:29:09,752 TRACE [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,752 DEBUG [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,752 TRACE [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,752 DEBUG [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=137 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,752 WARN [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=137, retrying... 2020-05-21 17:29:09,853 TRACE [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,853 DEBUG [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,853 TRACE [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,853 DEBUG [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=138 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,853 WARN [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=138, retrying... 2020-05-21 17:29:09,954 TRACE [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:09,954 DEBUG [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:09,954 TRACE [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:09,954 DEBUG [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=139 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:09,954 WARN [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=139, retrying... 2020-05-21 17:29:10,055 TRACE [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,055 TRACE [RSProcedureDispatcher-pool3-t43] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:10,059 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:10,060 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:10,060 DEBUG [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=140 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:10,062 WARN [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=140, retrying... 2020-05-21 17:29:10,130 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner, timeout=1590082150130, delay=0 2020-05-21 17:29:10,130 TRACE [ProcExecTimeout] procedure2.CompletedProcedureCleaner: Evict completed pid=2519, state=SUCCESS; ServerCrashProcedure server=regionserver-2.hbase.hbase.svc.cluster.local,16020,1590081637367, splitWal=true, meta=false 2020-05-21 17:29:10,130 TRACE [ProcExecTimeout] wal.WALProcedureStore: Delete 2519 2020-05-21 17:29:10,162 TRACE [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,162 DEBUG [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,162 TRACE [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,162 DEBUG [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=141 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,163 WARN [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=141, retrying... 2020-05-21 17:29:10,231 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync wait 100msec, slotIndex=1 , totalSynced=190.9K (1.3K/sec) 2020-05-21 17:29:10,240 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Sync slots=1/16, flushed=6 2020-05-21 17:29:10,240 TRACE [WALProcedureStoreSyncThread] wal.WALProcedureStore: Waiting for data. flushed=190.9K (1.3K/sec) 2020-05-21 17:29:10,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726625213250212 ns. 2020-05-21 17:29:10,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 86986 ns. 2020-05-21 17:29:10,263 TRACE [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,263 DEBUG [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,263 TRACE [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,263 DEBUG [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=142 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,264 WARN [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=142, retrying... 2020-05-21 17:29:10,364 TRACE [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,364 DEBUG [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,364 TRACE [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,365 DEBUG [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=143 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,365 WARN [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=143, retrying... 2020-05-21 17:29:10,465 TRACE [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,465 DEBUG [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,466 TRACE [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:10,466 DEBUG [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=144 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,466 WARN [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=144, retrying... 2020-05-21 17:29:10,566 TRACE [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,567 DEBUG [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,567 TRACE [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,567 DEBUG [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=145 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,567 WARN [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=145, retrying... 2020-05-21 17:29:10,668 TRACE [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,668 DEBUG [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,668 TRACE [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,668 DEBUG [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=146 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,669 WARN [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=146, retrying... 2020-05-21 17:29:10,725 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 5 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:10,725 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 5 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082160725 executing as hbase 2020-05-21 17:29:10,725 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:10,725 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:10,725 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 5 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082160725 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:10,769 TRACE [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,769 DEBUG [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,769 TRACE [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,769 DEBUG [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=147 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,770 WARN [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=147, retrying... 2020-05-21 17:29:10,870 TRACE [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,870 DEBUG [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,870 TRACE [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,870 DEBUG [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=148 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,871 WARN [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=148, retrying... 2020-05-21 17:29:10,971 TRACE [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:10,971 DEBUG [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:10,971 TRACE [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:10,972 DEBUG [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=149 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:10,972 WARN [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=149, retrying... 2020-05-21 17:29:11,072 TRACE [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,072 DEBUG [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,072 TRACE [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,073 DEBUG [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=150 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,073 WARN [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=150, retrying... 2020-05-21 17:29:11,173 TRACE [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,173 DEBUG [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,174 TRACE [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:11,174 DEBUG [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=151 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,174 WARN [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=151, retrying... 2020-05-21 17:29:11,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726626213247293 ns. 2020-05-21 17:29:11,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 100199 ns. 2020-05-21 17:29:11,274 TRACE [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,275 DEBUG [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,275 TRACE [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,276 DEBUG [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=152 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,276 WARN [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=152, retrying... 2020-05-21 17:29:11,376 TRACE [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,377 DEBUG [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,377 TRACE [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,377 DEBUG [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=153 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,377 WARN [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=153, retrying... 2020-05-21 17:29:11,477 TRACE [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,477 DEBUG [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,477 TRACE [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,477 DEBUG [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=154 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,478 WARN [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=154, retrying... 2020-05-21 17:29:11,578 TRACE [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,578 DEBUG [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,578 TRACE [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,578 DEBUG [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=155 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,579 WARN [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=155, retrying... 2020-05-21 17:29:11,679 TRACE [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,680 DEBUG [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,680 TRACE [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,680 DEBUG [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=156 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,681 WARN [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=156, retrying... 2020-05-21 17:29:11,782 TRACE [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,782 DEBUG [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,782 TRACE [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,782 DEBUG [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=157 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,783 WARN [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=157, retrying... 2020-05-21 17:29:11,883 TRACE [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,883 DEBUG [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,883 TRACE [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,883 DEBUG [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=158 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,884 WARN [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=158, retrying... 2020-05-21 17:29:11,984 TRACE [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:11,984 DEBUG [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:11,985 TRACE [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:11,985 DEBUG [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=159 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:11,985 WARN [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=159, retrying... 2020-05-21 17:29:12,085 TRACE [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,086 TRACE [RSProcedureDispatcher-pool3-t63] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:12,090 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:12,090 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:12,091 DEBUG [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=160 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:12,092 WARN [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=160, retrying... 2020-05-21 17:29:12,099 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 529 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 529 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082162099 executing as hbase 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:12,100 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:12,101 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 529 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082162099 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 888 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:12,193 TRACE [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,193 DEBUG [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,193 TRACE [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,193 DEBUG [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=161 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,194 WARN [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=161, retrying... 2020-05-21 17:29:12,199 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 527 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:29:12,199 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 527 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082162199 executing as hbase 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:12,200 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 527 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082162199 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 892 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:12,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726627213236831 ns. 2020-05-21 17:29:12,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 84023 ns. 2020-05-21 17:29:12,294 TRACE [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,294 DEBUG [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,294 TRACE [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,295 DEBUG [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=162 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,295 WARN [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=162, retrying... 2020-05-21 17:29:12,395 TRACE [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,396 DEBUG [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,396 TRACE [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,397 DEBUG [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=163 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,397 WARN [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=163, retrying... 2020-05-21 17:29:12,498 TRACE [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,498 DEBUG [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,498 TRACE [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,498 DEBUG [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=164 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,498 WARN [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=164, retrying... 2020-05-21 17:29:12,599 TRACE [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,599 DEBUG [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,600 TRACE [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:12,600 DEBUG [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=165 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,600 WARN [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=165, retrying... 2020-05-21 17:29:12,700 TRACE [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,701 DEBUG [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,701 TRACE [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,701 DEBUG [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=166 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,702 WARN [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=166, retrying... 2020-05-21 17:29:12,802 TRACE [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,802 DEBUG [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,802 TRACE [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,802 DEBUG [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=167 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,803 WARN [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=167, retrying... 2020-05-21 17:29:12,903 TRACE [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:12,903 DEBUG [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:12,903 TRACE [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:12,903 DEBUG [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=168 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:12,904 WARN [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=168, retrying... 2020-05-21 17:29:13,004 TRACE [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,004 DEBUG [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,005 TRACE [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:13,005 DEBUG [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=169 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,005 WARN [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=169, retrying... 2020-05-21 17:29:13,105 TRACE [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,106 DEBUG [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,106 TRACE [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,106 DEBUG [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=170 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,106 WARN [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=170, retrying... 2020-05-21 17:29:13,206 TRACE [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,207 DEBUG [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,207 TRACE [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,207 DEBUG [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=171 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,207 WARN [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=171, retrying... 2020-05-21 17:29:13,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726628213253325 ns. 2020-05-21 17:29:13,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 100379 ns. 2020-05-21 17:29:13,308 TRACE [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,308 DEBUG [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,308 TRACE [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,308 DEBUG [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=172 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,309 WARN [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=172, retrying... 2020-05-21 17:29:13,409 TRACE [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,409 DEBUG [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,410 TRACE [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:13,410 DEBUG [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=173 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,410 WARN [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=173, retrying... 2020-05-21 17:29:13,510 TRACE [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,510 DEBUG [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,510 TRACE [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,510 DEBUG [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=174 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,511 WARN [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=174, retrying... 2020-05-21 17:29:13,611 TRACE [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,611 DEBUG [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,611 TRACE [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,611 DEBUG [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=175 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,612 WARN [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=175, retrying... 2020-05-21 17:29:13,712 TRACE [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,712 DEBUG [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,712 TRACE [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,712 DEBUG [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=176 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,713 WARN [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=176, retrying... 2020-05-21 17:29:13,729 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 6 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:13,729 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 6 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082163729 executing as hbase 2020-05-21 17:29:13,730 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:13,730 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:13,730 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 6 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082163729 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:13,813 TRACE [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,813 DEBUG [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,813 TRACE [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,813 DEBUG [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=177 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,814 WARN [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=177, retrying... 2020-05-21 17:29:13,914 TRACE [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:13,915 DEBUG [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:13,915 TRACE [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:13,915 DEBUG [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=178 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:13,916 WARN [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=178, retrying... 2020-05-21 17:29:14,016 TRACE [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,017 DEBUG [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,017 TRACE [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,017 DEBUG [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=179 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,017 WARN [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=179, retrying... 2020-05-21 17:29:14,118 TRACE [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,118 TRACE [RSProcedureDispatcher-pool3-t83] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:14,126 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 8ms 2020-05-21 17:29:14,126 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:14,126 DEBUG [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=180 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:14,129 WARN [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=180, retrying... 2020-05-21 17:29:14,229 TRACE [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,229 DEBUG [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,229 TRACE [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,229 DEBUG [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=181 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,230 WARN [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=181, retrying... 2020-05-21 17:29:14,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726629213241963 ns. 2020-05-21 17:29:14,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 95821 ns. 2020-05-21 17:29:14,330 TRACE [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,331 DEBUG [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,331 TRACE [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:14,331 DEBUG [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=182 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,331 WARN [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=182, retrying... 2020-05-21 17:29:14,431 TRACE [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,431 DEBUG [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,431 TRACE [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,432 DEBUG [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=183 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,432 WARN [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=183, retrying... 2020-05-21 17:29:14,532 TRACE [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,532 DEBUG [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,533 TRACE [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,533 DEBUG [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=184 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,533 WARN [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=184, retrying... 2020-05-21 17:29:14,576 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082154576, delay=0 2020-05-21 17:29:14,633 TRACE [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,634 DEBUG [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,634 TRACE [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,634 DEBUG [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=185 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,634 WARN [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=185, retrying... 2020-05-21 17:29:14,735 TRACE [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,735 DEBUG [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,735 TRACE [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,735 DEBUG [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=186 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,736 WARN [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=186, retrying... 2020-05-21 17:29:14,836 TRACE [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,836 DEBUG [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,836 TRACE [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,836 DEBUG [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=187 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,837 WARN [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=187, retrying... 2020-05-21 17:29:14,937 TRACE [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:14,937 DEBUG [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:14,937 TRACE [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:14,938 DEBUG [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=188 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:14,938 WARN [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=188, retrying... 2020-05-21 17:29:15,038 TRACE [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,039 DEBUG [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,039 TRACE [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,039 DEBUG [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=189 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,039 WARN [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=189, retrying... 2020-05-21 17:29:15,103 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 530 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 530 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082165103 executing as hbase 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:15,104 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:15,105 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:15,105 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 530 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082165103 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 1089 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:29:15,139 TRACE [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,140 DEBUG [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,140 TRACE [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,140 DEBUG [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=190 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,140 WARN [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=190, retrying... 2020-05-21 17:29:15,203 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 528 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:29:15,203 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 528 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082165203 executing as hbase 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:15,204 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:15,205 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 528 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082165203 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 974 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:15,240 TRACE [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,241 DEBUG [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,241 TRACE [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,241 DEBUG [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=191 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,241 WARN [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=191, retrying... 2020-05-21 17:29:15,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726630213243227 ns. 2020-05-21 17:29:15,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 89819 ns. 2020-05-21 17:29:15,341 TRACE [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,342 DEBUG [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,342 TRACE [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,342 DEBUG [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=192 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,342 WARN [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=192, retrying... 2020-05-21 17:29:15,443 TRACE [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,443 DEBUG [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,443 TRACE [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,443 DEBUG [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=193 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,454 WARN [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=193, retrying... 2020-05-21 17:29:15,554 TRACE [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,555 DEBUG [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,555 TRACE [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,555 DEBUG [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=194 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,555 WARN [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=194, retrying... 2020-05-21 17:29:15,656 TRACE [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,656 DEBUG [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,656 TRACE [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,656 DEBUG [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=195 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,657 WARN [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=195, retrying... 2020-05-21 17:29:15,757 TRACE [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,757 DEBUG [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,757 TRACE [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,757 DEBUG [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=196 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,758 WARN [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=196, retrying... 2020-05-21 17:29:15,858 TRACE [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,859 DEBUG [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,859 TRACE [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:15,859 DEBUG [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=197 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,859 WARN [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=197, retrying... 2020-05-21 17:29:15,959 TRACE [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:15,960 DEBUG [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:15,960 TRACE [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:15,960 DEBUG [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=198 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:15,960 WARN [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=198, retrying... 2020-05-21 17:29:16,061 TRACE [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,061 DEBUG [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,061 TRACE [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,061 DEBUG [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=199 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,062 WARN [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=199, retrying... 2020-05-21 17:29:16,162 TRACE [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,162 TRACE [RSProcedureDispatcher-pool3-t103] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:16,166 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:16,167 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:16,167 DEBUG [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=200 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:16,169 WARN [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=200, retrying... 2020-05-21 17:29:16,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726631213247049 ns. 2020-05-21 17:29:16,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 105517 ns. 2020-05-21 17:29:16,269 TRACE [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,269 DEBUG [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,269 TRACE [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,270 DEBUG [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=201 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,270 WARN [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=201, retrying... 2020-05-21 17:29:16,371 TRACE [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,371 DEBUG [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,371 TRACE [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,371 DEBUG [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=202 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,372 WARN [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=202, retrying... 2020-05-21 17:29:16,472 TRACE [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,472 DEBUG [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,472 TRACE [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,472 DEBUG [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=203 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,473 WARN [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=203, retrying... 2020-05-21 17:29:16,573 TRACE [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,573 DEBUG [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,573 TRACE [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,573 DEBUG [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=204 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,574 WARN [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=204, retrying... 2020-05-21 17:29:16,674 TRACE [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,674 DEBUG [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,674 TRACE [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,674 DEBUG [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=205 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,675 WARN [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=205, retrying... 2020-05-21 17:29:16,733 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 7 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:16,733 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 7 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082166733 executing as hbase 2020-05-21 17:29:16,734 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:16,734 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:16,734 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 7 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082166733 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:16,775 TRACE [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,775 DEBUG [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,775 TRACE [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,775 DEBUG [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=206 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,776 WARN [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=206, retrying... 2020-05-21 17:29:16,876 TRACE [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,876 DEBUG [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,876 TRACE [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,876 DEBUG [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=207 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,877 WARN [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=207, retrying... 2020-05-21 17:29:16,977 TRACE [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:16,977 DEBUG [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:16,977 TRACE [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:16,978 DEBUG [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=208 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:16,978 WARN [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=208, retrying... 2020-05-21 17:29:17,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:29:17,079 TRACE [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,079 DEBUG [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,079 TRACE [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,092 DEBUG [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=209 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,092 WARN [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=209, retrying... 2020-05-21 17:29:17,192 TRACE [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,192 DEBUG [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,192 TRACE [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,192 DEBUG [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=210 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,193 WARN [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=210, retrying... 2020-05-21 17:29:17,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726632213243753 ns. 2020-05-21 17:29:17,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 88909 ns. 2020-05-21 17:29:17,293 TRACE [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,293 DEBUG [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,293 TRACE [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,293 DEBUG [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=211 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,294 WARN [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=211, retrying... 2020-05-21 17:29:17,394 TRACE [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,395 DEBUG [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,395 TRACE [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,395 DEBUG [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=212 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,395 WARN [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=212, retrying... 2020-05-21 17:29:17,496 TRACE [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,496 DEBUG [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,496 TRACE [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,496 DEBUG [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=213 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,497 WARN [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=213, retrying... 2020-05-21 17:29:17,597 TRACE [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,597 DEBUG [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,597 TRACE [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,598 DEBUG [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=214 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,598 WARN [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=214, retrying... 2020-05-21 17:29:17,698 TRACE [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,698 DEBUG [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,699 TRACE [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:17,699 DEBUG [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=215 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,699 WARN [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=215, retrying... 2020-05-21 17:29:17,800 TRACE [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,800 DEBUG [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,800 TRACE [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,800 DEBUG [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=216 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,801 WARN [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=216, retrying... 2020-05-21 17:29:17,901 TRACE [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:17,902 DEBUG [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:17,902 TRACE [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:17,902 DEBUG [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=217 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:17,902 WARN [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=217, retrying... 2020-05-21 17:29:18,002 TRACE [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,002 DEBUG [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,002 TRACE [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,002 DEBUG [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=218 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,003 WARN [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=218, retrying... 2020-05-21 17:29:18,103 TRACE [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,103 DEBUG [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,103 TRACE [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,103 DEBUG [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=219 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,104 WARN [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=219, retrying... 2020-05-21 17:29:18,107 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 531 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:18,108 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 531 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082168107 executing as hbase 2020-05-21 17:29:18,108 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,108 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,108 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:18,109 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 531 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082168107 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 844 } connection: 10.128.9.37:50537, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:29:18,204 TRACE [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,205 TRACE [RSProcedureDispatcher-pool3-t123] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:18,208 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 529 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:29:18,208 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 529 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082168208 executing as hbase 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:18,209 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:18,210 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 529 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082168208 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 862 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:18,210 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:29:18,210 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:18,210 DEBUG [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=220 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:18,213 WARN [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=220, retrying... 2020-05-21 17:29:18,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726633213218806 ns. 2020-05-21 17:29:18,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 85281 ns. 2020-05-21 17:29:18,313 TRACE [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,313 DEBUG [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,314 TRACE [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:18,328 DEBUG [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=221 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,328 WARN [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=221, retrying... 2020-05-21 17:29:18,428 TRACE [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,428 DEBUG [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,428 TRACE [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,428 DEBUG [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=222 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,428 WARN [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=222, retrying... 2020-05-21 17:29:18,529 TRACE [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,529 DEBUG [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,529 TRACE [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,529 DEBUG [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=223 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,529 WARN [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=223, retrying... 2020-05-21 17:29:18,630 TRACE [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,630 DEBUG [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,630 TRACE [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,630 DEBUG [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=224 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,630 WARN [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=224, retrying... 2020-05-21 17:29:18,731 TRACE [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,731 DEBUG [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,731 TRACE [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,731 DEBUG [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=225 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,731 WARN [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=225, retrying... 2020-05-21 17:29:18,832 TRACE [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,832 DEBUG [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,832 TRACE [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:18,832 DEBUG [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=226 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,833 WARN [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=226, retrying... 2020-05-21 17:29:18,933 TRACE [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:18,933 DEBUG [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:18,934 TRACE [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:18,934 DEBUG [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=227 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:18,934 WARN [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=227, retrying... 2020-05-21 17:29:19,034 TRACE [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,034 DEBUG [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,034 TRACE [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,034 DEBUG [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=228 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,035 WARN [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=228, retrying... 2020-05-21 17:29:19,135 TRACE [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,135 DEBUG [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,135 TRACE [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,135 DEBUG [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=229 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,135 WARN [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=229, retrying... 2020-05-21 17:29:19,236 TRACE [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,236 DEBUG [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,236 TRACE [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,236 DEBUG [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=230 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,237 WARN [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=230, retrying... 2020-05-21 17:29:19,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726634213252823 ns. 2020-05-21 17:29:19,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 75699 ns. 2020-05-21 17:29:19,337 TRACE [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,337 DEBUG [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,337 TRACE [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,337 DEBUG [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=231 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,338 WARN [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=231, retrying... 2020-05-21 17:29:19,438 TRACE [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,438 DEBUG [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,438 TRACE [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,438 DEBUG [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=232 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,439 WARN [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=232, retrying... 2020-05-21 17:29:19,539 TRACE [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,539 DEBUG [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,539 TRACE [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,539 DEBUG [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=233 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,540 WARN [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=233, retrying... 2020-05-21 17:29:19,576 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082159576, delay=0 2020-05-21 17:29:19,640 TRACE [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,640 DEBUG [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,640 TRACE [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,641 DEBUG [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=234 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,641 WARN [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=234, retrying... 2020-05-21 17:29:19,737 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 8 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:19,737 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 8 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082169737 executing as hbase 2020-05-21 17:29:19,738 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:19,738 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:19,738 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 8 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082169737 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:19,741 TRACE [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,741 DEBUG [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,741 TRACE [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,742 DEBUG [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=235 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,742 WARN [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=235, retrying... 2020-05-21 17:29:19,842 TRACE [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,842 DEBUG [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,842 TRACE [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,843 DEBUG [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=236 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,843 WARN [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=236, retrying... 2020-05-21 17:29:19,943 TRACE [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:19,944 DEBUG [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:19,944 TRACE [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:19,944 DEBUG [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=237 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:19,944 WARN [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=237, retrying... 2020-05-21 17:29:20,044 TRACE [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,045 DEBUG [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,045 TRACE [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,046 DEBUG [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=238 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,050 WARN [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=238, retrying... 2020-05-21 17:29:20,150 TRACE [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,151 DEBUG [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,151 TRACE [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,151 DEBUG [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=239 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,151 WARN [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=239, retrying... 2020-05-21 17:29:20,251 TRACE [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,251 TRACE [RSProcedureDispatcher-pool3-t15] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:20,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726635213221803 ns. 2020-05-21 17:29:20,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 69795 ns. 2020-05-21 17:29:20,257 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:29:20,257 DEBUG [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=240 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:20,259 WARN [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=240, retrying... 2020-05-21 17:29:20,257 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:20,360 TRACE [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,360 DEBUG [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,360 TRACE [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,360 DEBUG [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=241 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,360 WARN [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=241, retrying... 2020-05-21 17:29:20,460 TRACE [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,460 DEBUG [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,460 TRACE [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,460 DEBUG [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=242 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,460 WARN [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=242, retrying... 2020-05-21 17:29:20,561 TRACE [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,561 DEBUG [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,561 TRACE [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,561 DEBUG [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=243 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,561 WARN [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=243, retrying... 2020-05-21 17:29:20,662 TRACE [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,662 DEBUG [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,662 TRACE [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,662 DEBUG [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=244 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,662 WARN [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=244, retrying... 2020-05-21 17:29:20,763 TRACE [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,763 DEBUG [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,763 TRACE [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,763 DEBUG [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=245 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,764 WARN [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=245, retrying... 2020-05-21 17:29:20,864 TRACE [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,865 DEBUG [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,865 TRACE [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:20,865 DEBUG [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=246 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,865 WARN [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=246, retrying... 2020-05-21 17:29:20,966 TRACE [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:20,966 DEBUG [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:20,966 TRACE [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:20,966 DEBUG [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=247 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:20,966 WARN [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=247, retrying... 2020-05-21 17:29:21,067 TRACE [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,067 DEBUG [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,067 TRACE [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,067 DEBUG [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=248 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,067 WARN [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=248, retrying... 2020-05-21 17:29:21,112 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 532 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:21,113 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 532 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082171112 executing as hbase 2020-05-21 17:29:21,113 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,113 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,113 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,114 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:21,115 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 532 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082171112 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 844 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 3 totalTime: 3 2020-05-21 17:29:21,168 TRACE [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,168 DEBUG [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,168 TRACE [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,168 DEBUG [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=249 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,168 WARN [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=249, retrying... 2020-05-21 17:29:21,212 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 530 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4137 bytes 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 530 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082171213 executing as hbase 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,213 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:21,214 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 530 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082171213 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 862 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:21,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726636213250961 ns. 2020-05-21 17:29:21,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 119456 ns. 2020-05-21 17:29:21,269 TRACE [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,269 DEBUG [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,269 TRACE [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,269 DEBUG [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=250 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,269 WARN [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=250, retrying... 2020-05-21 17:29:21,370 TRACE [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,370 DEBUG [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,370 TRACE [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,370 DEBUG [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=251 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,371 WARN [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=251, retrying... 2020-05-21 17:29:21,471 TRACE [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,471 DEBUG [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,471 TRACE [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,471 DEBUG [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=252 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,471 WARN [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=252, retrying... 2020-05-21 17:29:21,572 TRACE [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,572 DEBUG [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,572 TRACE [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,572 DEBUG [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=253 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,573 WARN [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=253, retrying... 2020-05-21 17:29:21,673 TRACE [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,673 DEBUG [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,673 TRACE [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,674 DEBUG [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=254 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,674 WARN [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=254, retrying... 2020-05-21 17:29:21,774 TRACE [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,774 DEBUG [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,775 TRACE [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:21,775 DEBUG [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=255 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,775 WARN [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=255, retrying... 2020-05-21 17:29:21,876 TRACE [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,876 DEBUG [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,876 TRACE [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:21,876 DEBUG [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=256 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,876 WARN [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=256, retrying... 2020-05-21 17:29:21,977 TRACE [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:21,977 DEBUG [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:21,978 TRACE [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:21,981 DEBUG [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=257 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:21,981 WARN [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=257, retrying... 2020-05-21 17:29:22,082 TRACE [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,082 DEBUG [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,082 TRACE [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,083 DEBUG [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=258 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,083 WARN [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=258, retrying... 2020-05-21 17:29:22,183 TRACE [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,184 DEBUG [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,184 TRACE [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,184 DEBUG [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=259 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,184 WARN [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=259, retrying... 2020-05-21 17:29:22,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726637213258876 ns. 2020-05-21 17:29:22,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 95125 ns. 2020-05-21 17:29:22,284 TRACE [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,285 TRACE [RSProcedureDispatcher-pool3-t35] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:22,289 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:22,290 DEBUG [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=260 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:22,293 WARN [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=260, retrying... 2020-05-21 17:29:22,290 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:22,393 TRACE [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,393 DEBUG [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,393 TRACE [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,394 DEBUG [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=261 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,395 WARN [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=261, retrying... 2020-05-21 17:29:22,495 TRACE [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,496 DEBUG [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,496 TRACE [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:22,496 DEBUG [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=262 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,497 WARN [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=262, retrying... 2020-05-21 17:29:22,597 TRACE [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,597 DEBUG [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,597 TRACE [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,597 DEBUG [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=263 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,598 WARN [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=263, retrying... 2020-05-21 17:29:22,698 TRACE [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,698 DEBUG [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,698 TRACE [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,698 DEBUG [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=264 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,699 WARN [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=264, retrying... 2020-05-21 17:29:22,742 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 9 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:22,743 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 9 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082172743 executing as hbase 2020-05-21 17:29:22,743 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:22,743 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:22,743 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 9 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082172743 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:22,799 TRACE [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,799 DEBUG [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,799 TRACE [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,799 DEBUG [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=265 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,800 WARN [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=265, retrying... 2020-05-21 17:29:22,900 TRACE [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:22,900 DEBUG [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:22,900 TRACE [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:22,901 DEBUG [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=266 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:22,901 WARN [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=266, retrying... 2020-05-21 17:29:23,001 TRACE [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,002 DEBUG [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,002 TRACE [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,002 DEBUG [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=267 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,002 WARN [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=267, retrying... 2020-05-21 17:29:23,103 TRACE [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,103 DEBUG [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,103 TRACE [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,103 DEBUG [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=268 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,103 WARN [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=268, retrying... 2020-05-21 17:29:23,203 TRACE [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,204 DEBUG [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,204 TRACE [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,204 DEBUG [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=269 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,204 WARN [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=269, retrying... 2020-05-21 17:29:23,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726638213247248 ns. 2020-05-21 17:29:23,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 101391 ns. 2020-05-21 17:29:23,305 TRACE [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,305 DEBUG [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,305 TRACE [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,305 DEBUG [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=270 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,305 WARN [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=270, retrying... 2020-05-21 17:29:23,406 TRACE [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,406 DEBUG [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,406 TRACE [RSProcedureDispatcher-pool3-t46] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,406 DEBUG [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=271 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,406 WARN [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=271, retrying... 2020-05-21 17:29:23,507 TRACE [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,507 DEBUG [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,507 TRACE [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,507 DEBUG [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=272 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,507 WARN [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=272, retrying... 2020-05-21 17:29:23,608 TRACE [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,608 DEBUG [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,608 TRACE [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,608 DEBUG [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=273 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,609 WARN [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=273, retrying... 2020-05-21 17:29:23,709 TRACE [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,709 DEBUG [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,709 TRACE [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,709 DEBUG [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=274 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,710 WARN [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=274, retrying... 2020-05-21 17:29:23,810 TRACE [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,810 DEBUG [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,810 TRACE [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,811 DEBUG [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=275 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,812 WARN [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=275, retrying... 2020-05-21 17:29:23,912 TRACE [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:23,912 DEBUG [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:23,912 TRACE [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:23,913 DEBUG [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=276 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:23,913 WARN [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=276, retrying... 2020-05-21 17:29:24,013 TRACE [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,013 DEBUG [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,014 TRACE [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:24,014 DEBUG [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=277 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,014 WARN [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=277, retrying... 2020-05-21 17:29:24,114 TRACE [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,115 DEBUG [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,115 TRACE [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,115 DEBUG [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=278 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,115 WARN [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=278, retrying... 2020-05-21 17:29:24,118 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 533 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 533 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082174118 executing as hbase 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,118 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:24,119 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 533 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082174118 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 938 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:24,215 TRACE [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,216 DEBUG [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,216 TRACE [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,216 DEBUG [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=279 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,216 WARN [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=279, retrying... 2020-05-21 17:29:24,217 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 531 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4138 bytes 2020-05-21 17:29:24,217 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 531 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082174217 executing as hbase 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:24,218 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 531 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082174217 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 886 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:24,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726639213265243 ns. 2020-05-21 17:29:24,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 93355 ns. 2020-05-21 17:29:24,316 TRACE [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,316 TRACE [RSProcedureDispatcher-pool3-t55] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:24,320 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:24,321 DEBUG [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=280 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:24,323 WARN [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=280, retrying... 2020-05-21 17:29:24,321 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:24,423 TRACE [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,423 DEBUG [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,423 TRACE [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,424 DEBUG [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=281 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,424 WARN [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=281, retrying... 2020-05-21 17:29:24,524 TRACE [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,524 DEBUG [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,524 TRACE [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,524 DEBUG [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=282 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,525 WARN [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=282, retrying... 2020-05-21 17:29:24,576 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082164576, delay=0 2020-05-21 17:29:24,625 TRACE [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,625 DEBUG [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,625 TRACE [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,626 DEBUG [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=283 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,626 WARN [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=283, retrying... 2020-05-21 17:29:24,726 TRACE [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,726 DEBUG [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,727 TRACE [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:24,727 DEBUG [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=284 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,727 WARN [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=284, retrying... 2020-05-21 17:29:24,827 TRACE [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,828 DEBUG [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,828 TRACE [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:24,828 DEBUG [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=285 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,828 WARN [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=285, retrying... 2020-05-21 17:29:24,928 TRACE [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:24,929 DEBUG [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:24,929 TRACE [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:24,929 DEBUG [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=286 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:24,929 WARN [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=286, retrying... 2020-05-21 17:29:24,987 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 534 method_name: "ReportFileArchival" request_param: true priority: 0 timeout: 10000 totalRequestSize: 216 bytes 2020-05-21 17:29:24,987 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 534 service: RegionServerStatusService methodName: ReportFileArchival size: 216 connection: 10.128.9.37:50537 deadline: 1590082174987 executing as hbase 2020-05-21 17:29:24,987 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 534 service: RegionServerStatusService methodName: ReportFileArchival size: 216 connection: 10.128.9.37:50537 deadline: 1590082174987 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$FileArchiveNotificationRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:24,990 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 535 method_name: "ReportFileArchival" request_param: true priority: 0 timeout: 10000 totalRequestSize: 217 bytes 2020-05-21 17:29:24,990 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 535 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.9.37:50537 deadline: 1590082174990 executing as hbase 2020-05-21 17:29:24,990 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 535 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.9.37:50537 deadline: 1590082174990 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$FileArchiveNotificationRequest connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:25,029 TRACE [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,030 DEBUG [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,030 TRACE [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,030 DEBUG [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=287 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,030 WARN [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=287, retrying... 2020-05-21 17:29:25,130 TRACE [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,131 DEBUG [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,131 TRACE [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,131 DEBUG [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=288 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,131 WARN [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=288, retrying... 2020-05-21 17:29:25,180 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 532 method_name: "ReportFileArchival" request_param: true priority: 0 timeout: 10000 totalRequestSize: 217 bytes 2020-05-21 17:29:25,181 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 532 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.53.53:44503 deadline: 1590082175180 executing as hbase 2020-05-21 17:29:25,181 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 532 service: RegionServerStatusService methodName: ReportFileArchival size: 217 connection: 10.128.53.53:44503 deadline: 1590082175180 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$FileArchiveNotificationRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 0 totalTime: 1 2020-05-21 17:29:25,193 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 533 method_name: "ReportFileArchival" request_param: true priority: 0 timeout: 10000 totalRequestSize: 278 bytes 2020-05-21 17:29:25,194 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 533 service: RegionServerStatusService methodName: ReportFileArchival size: 278 connection: 10.128.53.53:44503 deadline: 1590082175193 executing as hbase 2020-05-21 17:29:25,194 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 533 service: RegionServerStatusService methodName: ReportFileArchival size: 278 connection: 10.128.53.53:44503 deadline: 1590082175193 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$FileArchiveNotificationRequest connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 0 totalTime: 1 2020-05-21 17:29:25,232 TRACE [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,232 DEBUG [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,232 TRACE [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,232 DEBUG [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=289 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,232 WARN [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=289, retrying... 2020-05-21 17:29:25,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726640213238859 ns. 2020-05-21 17:29:25,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 107022 ns. 2020-05-21 17:29:25,332 TRACE [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,333 DEBUG [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,333 TRACE [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,333 DEBUG [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=290 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,333 WARN [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=290, retrying... 2020-05-21 17:29:25,434 TRACE [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,434 DEBUG [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,434 TRACE [RSProcedureDispatcher-pool3-t66] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,434 DEBUG [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=291 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,435 WARN [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=291, retrying... 2020-05-21 17:29:25,535 TRACE [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,535 DEBUG [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,535 TRACE [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,535 DEBUG [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=292 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,536 WARN [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=292, retrying... 2020-05-21 17:29:25,636 TRACE [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,636 DEBUG [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,636 TRACE [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,636 DEBUG [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=293 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,637 WARN [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=293, retrying... 2020-05-21 17:29:25,737 TRACE [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,737 DEBUG [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,737 TRACE [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,737 DEBUG [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=294 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,738 WARN [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=294, retrying... 2020-05-21 17:29:25,747 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 10 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:25,747 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 10 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082175747 executing as hbase 2020-05-21 17:29:25,747 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:25,747 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:25,748 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 10 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082175747 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:25,838 TRACE [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,838 DEBUG [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,839 TRACE [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:25,839 DEBUG [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=295 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,840 WARN [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=295, retrying... 2020-05-21 17:29:25,940 TRACE [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:25,940 DEBUG [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:25,940 TRACE [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:25,940 DEBUG [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=296 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:25,941 WARN [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=296, retrying... 2020-05-21 17:29:26,041 TRACE [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,041 DEBUG [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,041 TRACE [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,041 DEBUG [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=297 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,042 WARN [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=297, retrying... 2020-05-21 17:29:26,142 TRACE [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,142 DEBUG [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,142 TRACE [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,143 DEBUG [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=298 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,143 WARN [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=298, retrying... 2020-05-21 17:29:26,243 TRACE [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,244 DEBUG [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,244 TRACE [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:26,244 DEBUG [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=299 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,244 WARN [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=299, retrying... 2020-05-21 17:29:26,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726641213246986 ns. 2020-05-21 17:29:26,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 85760 ns. 2020-05-21 17:29:26,344 TRACE [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,345 TRACE [RSProcedureDispatcher-pool3-t75] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:26,349 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:26,349 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:26,349 DEBUG [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=300 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:26,351 WARN [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=300, retrying... 2020-05-21 17:29:26,451 TRACE [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,451 DEBUG [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,451 TRACE [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,451 DEBUG [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=301 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,452 WARN [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=301, retrying... 2020-05-21 17:29:26,552 TRACE [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,552 DEBUG [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,552 TRACE [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,552 DEBUG [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=302 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,553 WARN [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=302, retrying... 2020-05-21 17:29:26,653 TRACE [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,653 DEBUG [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,654 TRACE [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:26,654 DEBUG [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=303 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,654 WARN [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=303, retrying... 2020-05-21 17:29:26,754 TRACE [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,754 DEBUG [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,755 TRACE [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:26,755 DEBUG [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=304 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,755 WARN [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=304, retrying... 2020-05-21 17:29:26,855 TRACE [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,856 DEBUG [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,856 TRACE [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,856 DEBUG [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=305 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,856 WARN [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=305, retrying... 2020-05-21 17:29:26,956 TRACE [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:26,957 DEBUG [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:26,957 TRACE [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:26,957 DEBUG [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=306 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:26,958 WARN [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=306, retrying... 2020-05-21 17:29:27,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:29:27,058 TRACE [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,058 DEBUG [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,059 TRACE [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:27,059 DEBUG [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=307 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,059 WARN [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=307, retrying... 2020-05-21 17:29:27,122 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 536 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:27,122 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 536 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082177122 executing as hbase 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:27,123 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:27,124 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 536 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082177122 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 938 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:29:27,159 TRACE [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,159 DEBUG [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,159 TRACE [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,160 DEBUG [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=308 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,160 WARN [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=308, retrying... 2020-05-21 17:29:27,221 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 534 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:27,221 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 534 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082177221 executing as hbase 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:27,222 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 534 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082177221 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 886 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:27,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726642213233787 ns. 2020-05-21 17:29:27,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 82680 ns. 2020-05-21 17:29:27,260 TRACE [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,260 DEBUG [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,260 TRACE [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,261 DEBUG [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=309 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,261 WARN [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=309, retrying... 2020-05-21 17:29:27,361 TRACE [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,361 DEBUG [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,361 TRACE [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,362 DEBUG [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=310 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,362 WARN [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=310, retrying... 2020-05-21 17:29:27,462 TRACE [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,462 DEBUG [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,463 TRACE [RSProcedureDispatcher-pool3-t86] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:27,463 DEBUG [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=311 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,463 WARN [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=311, retrying... 2020-05-21 17:29:27,563 TRACE [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,563 DEBUG [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,563 TRACE [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,564 DEBUG [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=312 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,564 WARN [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=312, retrying... 2020-05-21 17:29:27,664 TRACE [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,665 DEBUG [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,665 TRACE [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,665 DEBUG [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=313 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,665 WARN [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=313, retrying... 2020-05-21 17:29:27,765 TRACE [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,766 DEBUG [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,767 TRACE [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:27,767 DEBUG [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=314 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,767 WARN [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=314, retrying... 2020-05-21 17:29:27,867 TRACE [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,868 DEBUG [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,868 TRACE [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,868 DEBUG [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=315 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,868 WARN [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=315, retrying... 2020-05-21 17:29:27,969 TRACE [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:27,969 DEBUG [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:27,969 TRACE [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:27,969 DEBUG [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=316 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:27,970 WARN [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=316, retrying... 2020-05-21 17:29:28,070 TRACE [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,070 DEBUG [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,071 TRACE [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,071 DEBUG [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=317 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,071 WARN [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=317, retrying... 2020-05-21 17:29:28,171 TRACE [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,172 DEBUG [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,172 TRACE [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,172 DEBUG [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=318 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,173 WARN [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=318, retrying... 2020-05-21 17:29:28,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726643213221965 ns. 2020-05-21 17:29:28,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 111279 ns. 2020-05-21 17:29:28,273 TRACE [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,273 DEBUG [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,273 TRACE [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,273 DEBUG [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=319 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,273 WARN [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=319, retrying... 2020-05-21 17:29:28,374 TRACE [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,374 TRACE [RSProcedureDispatcher-pool3-t95] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:28,379 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:28,379 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:28,379 DEBUG [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=320 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:28,381 WARN [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=320, retrying... 2020-05-21 17:29:28,481 TRACE [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,481 DEBUG [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,481 TRACE [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,482 DEBUG [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=321 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,482 WARN [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=321, retrying... 2020-05-21 17:29:28,582 TRACE [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,582 DEBUG [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,583 TRACE [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:28,583 DEBUG [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=322 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,583 WARN [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=322, retrying... 2020-05-21 17:29:28,683 TRACE [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,684 DEBUG [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,684 TRACE [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,684 DEBUG [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=323 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,684 WARN [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=323, retrying... 2020-05-21 17:29:28,751 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 11 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:28,751 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 11 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082178751 executing as hbase 2020-05-21 17:29:28,752 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:28,752 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:28,752 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 11 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082178751 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:28,784 TRACE [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,785 DEBUG [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,785 TRACE [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:28,785 DEBUG [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=324 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,786 WARN [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=324, retrying... 2020-05-21 17:29:28,886 TRACE [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,886 DEBUG [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,886 TRACE [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,886 DEBUG [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=325 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,887 WARN [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=325, retrying... 2020-05-21 17:29:28,987 TRACE [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:28,987 DEBUG [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:28,987 TRACE [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:28,987 DEBUG [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=326 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:28,988 WARN [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=326, retrying... 2020-05-21 17:29:29,088 TRACE [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,088 DEBUG [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,088 TRACE [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,088 DEBUG [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=327 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,089 WARN [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=327, retrying... 2020-05-21 17:29:29,189 TRACE [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,189 DEBUG [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,189 TRACE [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,190 DEBUG [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=328 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,190 WARN [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=328, retrying... 2020-05-21 17:29:29,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726644213279882 ns. 2020-05-21 17:29:29,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 78137 ns. 2020-05-21 17:29:29,290 TRACE [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,290 DEBUG [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,290 TRACE [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,290 DEBUG [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=329 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,291 WARN [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=329, retrying... 2020-05-21 17:29:29,391 TRACE [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,391 DEBUG [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,391 TRACE [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,391 DEBUG [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=330 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,392 WARN [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=330, retrying... 2020-05-21 17:29:29,492 TRACE [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,492 DEBUG [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,493 TRACE [RSProcedureDispatcher-pool3-t106] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:29,493 DEBUG [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=331 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,493 WARN [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=331, retrying... 2020-05-21 17:29:29,577 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082169577, delay=0 2020-05-21 17:29:29,593 TRACE [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,594 DEBUG [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,594 TRACE [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:29,594 DEBUG [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=332 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,594 WARN [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=332, retrying... 2020-05-21 17:29:29,694 TRACE [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,695 DEBUG [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,695 TRACE [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,695 DEBUG [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=333 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,695 WARN [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=333, retrying... 2020-05-21 17:29:29,796 TRACE [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,796 DEBUG [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,796 TRACE [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,796 DEBUG [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=334 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,796 WARN [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=334, retrying... 2020-05-21 17:29:29,896 TRACE [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,896 DEBUG [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,896 TRACE [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,896 DEBUG [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=335 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,897 WARN [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=335, retrying... 2020-05-21 17:29:29,997 TRACE [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:29,997 DEBUG [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:29,997 TRACE [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:29,997 DEBUG [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=336 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:29,998 WARN [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=336, retrying... 2020-05-21 17:29:30,098 TRACE [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,098 DEBUG [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,110 TRACE [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 12ms 2020-05-21 17:29:30,110 DEBUG [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=337 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,111 WARN [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=337, retrying... 2020-05-21 17:29:30,126 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 537 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:30,126 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 537 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082180126 executing as hbase 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:30,127 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:30,128 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 537 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082180126 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 949 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:30,211 TRACE [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,211 DEBUG [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,211 TRACE [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,211 DEBUG [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=338 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,212 WARN [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=338, retrying... 2020-05-21 17:29:30,225 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 535 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 535 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082180225 executing as hbase 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:30,225 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:30,226 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 535 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082180225 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 965 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:30,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726645213244545 ns. 2020-05-21 17:29:30,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 77406 ns. 2020-05-21 17:29:30,312 TRACE [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,312 DEBUG [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,312 TRACE [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,312 DEBUG [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=339 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,313 WARN [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=339, retrying... 2020-05-21 17:29:30,413 TRACE [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,413 TRACE [RSProcedureDispatcher-pool3-t115] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:30,418 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:30,418 DEBUG [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=340 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:30,419 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:30,420 WARN [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=340, retrying... 2020-05-21 17:29:30,520 TRACE [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,521 DEBUG [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,521 TRACE [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,521 DEBUG [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=341 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,521 WARN [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=341, retrying... 2020-05-21 17:29:30,622 TRACE [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,622 DEBUG [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,622 TRACE [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,623 DEBUG [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=342 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,624 WARN [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=342, retrying... 2020-05-21 17:29:30,724 TRACE [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,724 DEBUG [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,725 TRACE [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:30,725 DEBUG [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=343 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,725 WARN [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=343, retrying... 2020-05-21 17:29:30,826 TRACE [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,826 DEBUG [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,826 TRACE [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,826 DEBUG [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=344 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,826 WARN [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=344, retrying... 2020-05-21 17:29:30,927 TRACE [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:30,927 DEBUG [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:30,927 TRACE [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:30,927 DEBUG [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=345 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:30,927 WARN [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=345, retrying... 2020-05-21 17:29:31,028 TRACE [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,028 DEBUG [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,028 TRACE [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,028 DEBUG [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=346 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,028 WARN [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=346, retrying... 2020-05-21 17:29:31,129 TRACE [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,129 DEBUG [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,129 TRACE [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,129 DEBUG [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=347 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,129 WARN [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=347, retrying... 2020-05-21 17:29:31,230 TRACE [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,230 DEBUG [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,230 TRACE [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,230 DEBUG [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=348 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,230 WARN [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=348, retrying... 2020-05-21 17:29:31,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726646213199883 ns. 2020-05-21 17:29:31,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 83484 ns. 2020-05-21 17:29:31,331 TRACE [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,331 DEBUG [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,331 TRACE [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,331 DEBUG [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=349 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,331 WARN [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=349, retrying... 2020-05-21 17:29:31,432 TRACE [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,432 DEBUG [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,432 TRACE [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,433 DEBUG [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=350 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,433 WARN [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=350, retrying... 2020-05-21 17:29:31,533 TRACE [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,533 DEBUG [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,534 TRACE [RSProcedureDispatcher-pool3-t126] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:31,534 DEBUG [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=351 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,534 WARN [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=351, retrying... 2020-05-21 17:29:31,634 TRACE [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,634 DEBUG [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,634 TRACE [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,635 DEBUG [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=352 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,635 WARN [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=352, retrying... 2020-05-21 17:29:31,735 TRACE [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,735 DEBUG [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,735 TRACE [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,736 DEBUG [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=353 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,736 WARN [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=353, retrying... 2020-05-21 17:29:31,755 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 12 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:31,755 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 12 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082181755 executing as hbase 2020-05-21 17:29:31,756 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:31,756 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:31,756 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 12 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082181755 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:31,836 TRACE [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,836 DEBUG [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,837 TRACE [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:31,837 DEBUG [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=354 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,837 WARN [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=354, retrying... 2020-05-21 17:29:31,937 TRACE [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:31,938 DEBUG [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:31,938 TRACE [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:31,938 DEBUG [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=355 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:31,939 WARN [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=355, retrying... 2020-05-21 17:29:32,039 TRACE [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,039 DEBUG [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,039 TRACE [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,040 DEBUG [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=356 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,040 WARN [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=356, retrying... 2020-05-21 17:29:32,140 TRACE [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,140 DEBUG [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,140 TRACE [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,140 DEBUG [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=357 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,141 WARN [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=357, retrying... 2020-05-21 17:29:32,241 TRACE [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,241 DEBUG [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,241 TRACE [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,242 DEBUG [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=358 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,242 WARN [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=358, retrying... 2020-05-21 17:29:32,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726647213220936 ns. 2020-05-21 17:29:32,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 102957 ns. 2020-05-21 17:29:32,342 TRACE [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,342 DEBUG [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,342 TRACE [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,343 DEBUG [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=359 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,343 WARN [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=359, retrying... 2020-05-21 17:29:32,443 TRACE [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,443 TRACE [RSProcedureDispatcher-pool3-t7] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:32,448 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:32,448 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:32,449 DEBUG [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=360 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:32,450 WARN [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=360, retrying... 2020-05-21 17:29:32,550 TRACE [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,551 DEBUG [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,551 TRACE [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,551 DEBUG [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=361 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,551 WARN [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=361, retrying... 2020-05-21 17:29:32,651 TRACE [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,652 DEBUG [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,652 TRACE [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,652 DEBUG [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=362 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,652 WARN [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=362, retrying... 2020-05-21 17:29:32,752 TRACE [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,753 DEBUG [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,753 TRACE [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,753 DEBUG [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=363 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,753 WARN [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=363, retrying... 2020-05-21 17:29:32,853 TRACE [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,854 DEBUG [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,854 TRACE [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,854 DEBUG [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=364 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,854 WARN [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=364, retrying... 2020-05-21 17:29:32,954 TRACE [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:32,955 DEBUG [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:32,955 TRACE [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:32,955 DEBUG [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=365 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:32,955 WARN [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=365, retrying... 2020-05-21 17:29:33,055 TRACE [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,055 DEBUG [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,055 TRACE [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,055 DEBUG [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=366 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,056 WARN [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=366, retrying... 2020-05-21 17:29:33,130 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 538 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:33,130 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 538 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082183130 executing as hbase 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:33,131 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:33,132 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:33,132 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 538 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082183130 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 904 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:29:33,156 TRACE [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,156 DEBUG [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,156 TRACE [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,156 DEBUG [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=367 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,157 WARN [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=367, retrying... 2020-05-21 17:29:33,228 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 536 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:33,229 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 536 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082183228 executing as hbase 2020-05-21 17:29:33,229 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:33,230 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 536 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082183228 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 832 } connection: 10.128.53.53:44503, response queueTime: 1 processingTime: 1 totalTime: 2 2020-05-21 17:29:33,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726648213255254 ns. 2020-05-21 17:29:33,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 94511 ns. 2020-05-21 17:29:33,257 TRACE [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,257 DEBUG [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,257 TRACE [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,257 DEBUG [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=368 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,258 WARN [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=368, retrying... 2020-05-21 17:29:33,358 TRACE [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,358 DEBUG [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,358 TRACE [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,358 DEBUG [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=369 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,359 WARN [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=369, retrying... 2020-05-21 17:29:33,459 TRACE [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,459 DEBUG [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,459 TRACE [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,459 DEBUG [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=370 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,460 WARN [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=370, retrying... 2020-05-21 17:29:33,560 TRACE [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,560 DEBUG [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,560 TRACE [RSProcedureDispatcher-pool3-t18] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,560 DEBUG [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=371 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,561 WARN [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=371, retrying... 2020-05-21 17:29:33,661 TRACE [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,661 DEBUG [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,661 TRACE [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,661 DEBUG [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=372 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,661 WARN [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=372, retrying... 2020-05-21 17:29:33,762 TRACE [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,762 DEBUG [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,762 TRACE [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,762 DEBUG [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=373 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,762 WARN [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=373, retrying... 2020-05-21 17:29:33,862 TRACE [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,862 DEBUG [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,862 TRACE [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,862 DEBUG [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=374 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,862 WARN [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=374, retrying... 2020-05-21 17:29:33,963 TRACE [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:33,963 DEBUG [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:33,963 TRACE [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:33,963 DEBUG [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=375 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:33,963 WARN [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=375, retrying... 2020-05-21 17:29:34,063 TRACE [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,064 DEBUG [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,064 TRACE [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,064 DEBUG [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=376 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,064 WARN [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=376, retrying... 2020-05-21 17:29:34,164 TRACE [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,165 DEBUG [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,165 TRACE [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,165 DEBUG [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=377 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,165 WARN [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=377, retrying... 2020-05-21 17:29:34,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726649213257146 ns. 2020-05-21 17:29:34,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 88208 ns. 2020-05-21 17:29:34,265 TRACE [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,266 DEBUG [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,266 TRACE [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,266 DEBUG [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=378 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,266 WARN [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=378, retrying... 2020-05-21 17:29:34,366 TRACE [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,366 DEBUG [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,367 TRACE [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:34,367 DEBUG [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=379 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,367 WARN [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=379, retrying... 2020-05-21 17:29:34,467 TRACE [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,467 TRACE [RSProcedureDispatcher-pool3-t27] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:34,473 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:29:34,473 DEBUG [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=380 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:34,478 WARN [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=380, retrying... 2020-05-21 17:29:34,473 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:34,577 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082174577, delay=0 2020-05-21 17:29:34,578 TRACE [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,578 DEBUG [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,578 TRACE [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,578 DEBUG [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=381 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,579 WARN [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=381, retrying... 2020-05-21 17:29:34,679 TRACE [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,679 DEBUG [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,679 TRACE [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,679 DEBUG [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=382 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,680 WARN [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=382, retrying... 2020-05-21 17:29:34,759 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 13 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:34,759 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 13 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082184759 executing as hbase 2020-05-21 17:29:34,760 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:34,760 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:34,760 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 13 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082184759 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:34,780 TRACE [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,780 DEBUG [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,780 TRACE [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,780 DEBUG [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=383 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,781 WARN [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=383, retrying... 2020-05-21 17:29:34,881 TRACE [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:34,881 DEBUG [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:34,881 TRACE [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:34,881 DEBUG [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=384 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:34,882 WARN [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=384, retrying... 2020-05-21 17:29:35,070 TRACE [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,070 DEBUG [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,070 TRACE [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,071 DEBUG [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=385 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,071 WARN [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=385, retrying... 2020-05-21 17:29:35,171 TRACE [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,172 DEBUG [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,172 TRACE [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,172 DEBUG [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=386 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,172 WARN [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=386, retrying... 2020-05-21 17:29:35,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726650213312460 ns. 2020-05-21 17:29:35,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 84285 ns. 2020-05-21 17:29:35,272 TRACE [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,273 DEBUG [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,273 TRACE [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,273 DEBUG [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=387 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,273 WARN [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=387, retrying... 2020-05-21 17:29:35,373 TRACE [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,373 DEBUG [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,374 TRACE [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,374 DEBUG [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=388 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,374 WARN [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=388, retrying... 2020-05-21 17:29:35,474 TRACE [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,474 DEBUG [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,474 TRACE [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,475 DEBUG [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=389 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,475 WARN [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=389, retrying... 2020-05-21 17:29:35,575 TRACE [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,575 DEBUG [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,575 TRACE [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,575 DEBUG [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=390 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,576 WARN [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=390, retrying... 2020-05-21 17:29:35,676 TRACE [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,676 DEBUG [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,676 TRACE [RSProcedureDispatcher-pool3-t38] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,676 DEBUG [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=391 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,677 WARN [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=391, retrying... 2020-05-21 17:29:35,777 TRACE [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,777 DEBUG [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,777 TRACE [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,777 DEBUG [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=392 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,778 WARN [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=392, retrying... 2020-05-21 17:29:35,878 TRACE [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,878 DEBUG [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,878 TRACE [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,878 DEBUG [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=393 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,879 WARN [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=393, retrying... 2020-05-21 17:29:35,979 TRACE [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:35,979 DEBUG [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:35,979 TRACE [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:35,979 DEBUG [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=394 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:35,980 WARN [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=394, retrying... 2020-05-21 17:29:36,080 TRACE [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,080 DEBUG [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,080 TRACE [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,080 DEBUG [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=395 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,081 WARN [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=395, retrying... 2020-05-21 17:29:36,134 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 539 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:36,134 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 539 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082186134 executing as hbase 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:36,135 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:36,136 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 539 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082186134 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 904 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 2 totalTime: 2 2020-05-21 17:29:36,181 TRACE [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,181 DEBUG [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,181 TRACE [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,181 DEBUG [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=396 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,182 WARN [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=396, retrying... 2020-05-21 17:29:36,233 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 537 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:36,233 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 537 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082186233 executing as hbase 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:36,234 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 537 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082186233 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 832 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:36,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726651213193472 ns. 2020-05-21 17:29:36,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 86492 ns. 2020-05-21 17:29:36,282 TRACE [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,282 DEBUG [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,282 TRACE [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,282 DEBUG [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=397 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,283 WARN [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=397, retrying... 2020-05-21 17:29:36,383 TRACE [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,383 DEBUG [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,383 TRACE [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,383 DEBUG [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=398 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,384 WARN [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=398, retrying... 2020-05-21 17:29:36,484 TRACE [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,484 TRACE [RSProcedureDispatcher-pool3-t46] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:36,490 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 6ms 2020-05-21 17:29:36,490 DEBUG [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=399 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:36,491 WARN [RSProcedureDispatcher-pool3-t46] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=399, retrying... 2020-05-21 17:29:36,491 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:36,591 TRACE [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,591 DEBUG [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,591 TRACE [RSProcedureDispatcher-pool3-t47] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,591 DEBUG [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=400 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,592 WARN [RSProcedureDispatcher-pool3-t47] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=400, retrying... 2020-05-21 17:29:36,692 TRACE [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,693 DEBUG [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,693 TRACE [RSProcedureDispatcher-pool3-t48] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,693 DEBUG [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=401 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,695 WARN [RSProcedureDispatcher-pool3-t48] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=401, retrying... 2020-05-21 17:29:36,795 TRACE [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,795 DEBUG [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,795 TRACE [RSProcedureDispatcher-pool3-t49] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,796 DEBUG [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=402 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,796 WARN [RSProcedureDispatcher-pool3-t49] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=402, retrying... 2020-05-21 17:29:36,896 TRACE [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,897 DEBUG [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,897 TRACE [RSProcedureDispatcher-pool3-t50] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,897 DEBUG [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=403 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,897 WARN [RSProcedureDispatcher-pool3-t50] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=403, retrying... 2020-05-21 17:29:36,997 TRACE [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:36,998 DEBUG [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:36,998 TRACE [RSProcedureDispatcher-pool3-t51] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:36,998 DEBUG [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=404 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:36,998 WARN [RSProcedureDispatcher-pool3-t51] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=404, retrying... 2020-05-21 17:29:37,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:29:37,099 TRACE [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,099 DEBUG [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,099 TRACE [RSProcedureDispatcher-pool3-t52] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,099 DEBUG [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=405 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,099 WARN [RSProcedureDispatcher-pool3-t52] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=405, retrying... 2020-05-21 17:29:37,200 TRACE [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,200 DEBUG [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,200 TRACE [RSProcedureDispatcher-pool3-t53] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,201 DEBUG [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=406 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,202 WARN [RSProcedureDispatcher-pool3-t53] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=406, retrying... 2020-05-21 17:29:37,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726652213246516 ns. 2020-05-21 17:29:37,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 99779 ns. 2020-05-21 17:29:37,302 TRACE [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,302 DEBUG [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,303 TRACE [RSProcedureDispatcher-pool3-t54] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:37,303 DEBUG [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=407 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,303 WARN [RSProcedureDispatcher-pool3-t54] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=407, retrying... 2020-05-21 17:29:37,403 TRACE [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,404 DEBUG [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,404 TRACE [RSProcedureDispatcher-pool3-t55] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,404 DEBUG [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=408 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,404 WARN [RSProcedureDispatcher-pool3-t55] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=408, retrying... 2020-05-21 17:29:37,504 TRACE [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,505 DEBUG [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,505 TRACE [RSProcedureDispatcher-pool3-t56] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,505 DEBUG [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=409 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,505 WARN [RSProcedureDispatcher-pool3-t56] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=409, retrying... 2020-05-21 17:29:37,606 TRACE [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,606 DEBUG [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,606 TRACE [RSProcedureDispatcher-pool3-t57] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,606 DEBUG [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=410 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,606 WARN [RSProcedureDispatcher-pool3-t57] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=410, retrying... 2020-05-21 17:29:37,706 TRACE [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,707 DEBUG [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,707 TRACE [RSProcedureDispatcher-pool3-t58] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,707 DEBUG [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=411 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,708 WARN [RSProcedureDispatcher-pool3-t58] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=411, retrying... 2020-05-21 17:29:37,763 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 14 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:37,763 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 14 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082187763 executing as hbase 2020-05-21 17:29:37,764 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:37,764 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:37,764 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 14 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082187763 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:37,808 TRACE [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,808 DEBUG [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,808 TRACE [RSProcedureDispatcher-pool3-t59] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,808 DEBUG [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=412 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,809 WARN [RSProcedureDispatcher-pool3-t59] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=412, retrying... 2020-05-21 17:29:37,909 TRACE [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:37,909 DEBUG [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:37,909 TRACE [RSProcedureDispatcher-pool3-t60] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:37,909 DEBUG [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=413 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:37,909 WARN [RSProcedureDispatcher-pool3-t60] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=413, retrying... 2020-05-21 17:29:37,970 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 19 method_name: "IsMasterRunning" request_param: true priority: 0 timeout: 60000 totalRequestSize: 29 bytes 2020-05-21 17:29:37,970 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 19 service: MasterService methodName: IsMasterRunning size: 29 connection: 10.128.25.30:47836 deadline: 1590082237970 executing as hbase 2020-05-21 17:29:37,970 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 19 service: MasterService methodName: IsMasterRunning size: 29 connection: 10.128.25.30:47836 deadline: 1590082237970 param: TODO: class org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$IsMasterRunningRequest connection: 10.128.25.30:47836, response is_master_running: true queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:37,972 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 20 method_name: "Balance" request_param: true priority: 0 timeout: 60000 totalRequestSize: 23 bytes 2020-05-21 17:29:37,973 TRACE [RpcServer.default.FPBQ.Fifo.handler=48,queue=3,port=16000] ipc.RpcServer: callId: 20 service: MasterService methodName: Balance size: 23 connection: 10.128.25.30:47836 deadline: 1590082237972 executing as hbase 2020-05-21 17:29:38,009 TRACE [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,009 DEBUG [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,009 TRACE [RSProcedureDispatcher-pool3-t61] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,009 DEBUG [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=414 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,009 WARN [RSProcedureDispatcher-pool3-t61] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=414, retrying... 2020-05-21 17:29:38,109 TRACE [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,110 DEBUG [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,110 TRACE [RSProcedureDispatcher-pool3-t62] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,110 DEBUG [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=415 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,110 WARN [RSProcedureDispatcher-pool3-t62] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=415, retrying... 2020-05-21 17:29:38,210 TRACE [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,211 DEBUG [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,211 TRACE [RSProcedureDispatcher-pool3-t63] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,211 DEBUG [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=416 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,212 WARN [RSProcedureDispatcher-pool3-t63] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=416, retrying... 2020-05-21 17:29:38,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726653213254474 ns. 2020-05-21 17:29:38,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 84979 ns. 2020-05-21 17:29:38,312 TRACE [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,312 DEBUG [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,312 TRACE [RSProcedureDispatcher-pool3-t64] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,312 DEBUG [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=417 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,312 WARN [RSProcedureDispatcher-pool3-t64] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=417, retrying... 2020-05-21 17:29:38,413 TRACE [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,413 DEBUG [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,413 TRACE [RSProcedureDispatcher-pool3-t65] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,413 DEBUG [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=418 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,413 WARN [RSProcedureDispatcher-pool3-t65] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=418, retrying... 2020-05-21 17:29:38,514 TRACE [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,514 TRACE [RSProcedureDispatcher-pool3-t66] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:38,518 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 4ms 2020-05-21 17:29:38,519 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:38,519 DEBUG [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=419 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:38,519 WARN [RSProcedureDispatcher-pool3-t66] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=419, retrying... 2020-05-21 17:29:38,619 TRACE [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,619 DEBUG [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,619 TRACE [RSProcedureDispatcher-pool3-t67] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,619 DEBUG [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=420 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,619 WARN [RSProcedureDispatcher-pool3-t67] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=420, retrying... 2020-05-21 17:29:38,720 TRACE [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,720 DEBUG [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,720 TRACE [RSProcedureDispatcher-pool3-t68] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,720 DEBUG [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=421 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,721 WARN [RSProcedureDispatcher-pool3-t68] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=421, retrying... 2020-05-21 17:29:38,821 TRACE [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,821 DEBUG [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,821 TRACE [RSProcedureDispatcher-pool3-t69] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,821 DEBUG [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=422 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,821 WARN [RSProcedureDispatcher-pool3-t69] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=422, retrying... 2020-05-21 17:29:38,922 TRACE [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:38,922 DEBUG [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:38,922 TRACE [RSProcedureDispatcher-pool3-t70] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:38,922 DEBUG [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=423 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:38,922 WARN [RSProcedureDispatcher-pool3-t70] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=423, retrying... 2020-05-21 17:29:39,023 TRACE [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,023 DEBUG [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,023 TRACE [RSProcedureDispatcher-pool3-t71] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,023 DEBUG [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=424 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,023 WARN [RSProcedureDispatcher-pool3-t71] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=424, retrying... 2020-05-21 17:29:39,123 TRACE [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,124 DEBUG [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,124 TRACE [RSProcedureDispatcher-pool3-t72] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,124 DEBUG [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=425 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,124 WARN [RSProcedureDispatcher-pool3-t72] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=425, retrying... 2020-05-21 17:29:39,138 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 540 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:39,138 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 540 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082189138 executing as hbase 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:39,139 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:39,140 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 540 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082189138 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 939 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:39,224 TRACE [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,225 DEBUG [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,225 TRACE [RSProcedureDispatcher-pool3-t73] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,225 DEBUG [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=426 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,225 WARN [RSProcedureDispatcher-pool3-t73] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=426, retrying... 2020-05-21 17:29:39,236 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 538 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 538 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082189237 executing as hbase 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,237 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:39,238 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:39,238 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 538 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082189237 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 942 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:39,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726654213242254 ns. 2020-05-21 17:29:39,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 90249 ns. 2020-05-21 17:29:39,325 TRACE [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,326 DEBUG [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,326 TRACE [RSProcedureDispatcher-pool3-t74] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,326 DEBUG [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=427 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,326 WARN [RSProcedureDispatcher-pool3-t74] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=427, retrying... 2020-05-21 17:29:39,426 TRACE [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,427 DEBUG [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,427 TRACE [RSProcedureDispatcher-pool3-t75] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,427 DEBUG [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=428 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,427 WARN [RSProcedureDispatcher-pool3-t75] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=428, retrying... 2020-05-21 17:29:39,527 TRACE [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,528 DEBUG [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,528 TRACE [RSProcedureDispatcher-pool3-t76] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,528 DEBUG [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=429 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,528 WARN [RSProcedureDispatcher-pool3-t76] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=429, retrying... 2020-05-21 17:29:39,577 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082179577, delay=0 2020-05-21 17:29:39,628 TRACE [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,628 DEBUG [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,628 TRACE [RSProcedureDispatcher-pool3-t77] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,629 DEBUG [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=430 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,629 WARN [RSProcedureDispatcher-pool3-t77] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=430, retrying... 2020-05-21 17:29:39,729 TRACE [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,730 DEBUG [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,730 TRACE [RSProcedureDispatcher-pool3-t78] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,730 DEBUG [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=431 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,730 WARN [RSProcedureDispatcher-pool3-t78] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=431, retrying... 2020-05-21 17:29:39,831 TRACE [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,831 DEBUG [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,831 TRACE [RSProcedureDispatcher-pool3-t79] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,831 DEBUG [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=432 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,831 WARN [RSProcedureDispatcher-pool3-t79] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=432, retrying... 2020-05-21 17:29:39,931 TRACE [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:39,932 DEBUG [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:39,932 TRACE [RSProcedureDispatcher-pool3-t80] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:39,932 DEBUG [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=433 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:39,932 WARN [RSProcedureDispatcher-pool3-t80] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=433, retrying... 2020-05-21 17:29:40,032 TRACE [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,032 DEBUG [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,033 TRACE [RSProcedureDispatcher-pool3-t81] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:40,033 DEBUG [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=434 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,033 WARN [RSProcedureDispatcher-pool3-t81] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=434, retrying... 2020-05-21 17:29:40,133 TRACE [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,133 DEBUG [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,134 TRACE [RSProcedureDispatcher-pool3-t82] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,134 DEBUG [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=435 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,134 WARN [RSProcedureDispatcher-pool3-t82] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=435, retrying... 2020-05-21 17:29:40,235 TRACE [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,235 DEBUG [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,235 TRACE [RSProcedureDispatcher-pool3-t83] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,235 DEBUG [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=436 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,235 WARN [RSProcedureDispatcher-pool3-t83] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=436, retrying... 2020-05-21 17:29:40,240 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner, timeout=1590082180240, delay=0 2020-05-21 17:29:40,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726655213227241 ns. 2020-05-21 17:29:40,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 93476 ns. 2020-05-21 17:29:40,336 TRACE [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,336 DEBUG [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,336 TRACE [RSProcedureDispatcher-pool3-t84] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,336 DEBUG [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=437 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,336 WARN [RSProcedureDispatcher-pool3-t84] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=437, retrying... 2020-05-21 17:29:40,436 TRACE [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,437 DEBUG [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,437 TRACE [RSProcedureDispatcher-pool3-t85] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,437 DEBUG [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=438 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,437 WARN [RSProcedureDispatcher-pool3-t85] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=438, retrying... 2020-05-21 17:29:40,537 TRACE [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,538 TRACE [RSProcedureDispatcher-pool3-t86] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:40,543 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:40,543 DEBUG [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=439 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:40,543 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:40,544 WARN [RSProcedureDispatcher-pool3-t86] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=439, retrying... 2020-05-21 17:29:40,644 TRACE [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,644 DEBUG [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,644 TRACE [RSProcedureDispatcher-pool3-t87] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,644 DEBUG [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=440 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,645 WARN [RSProcedureDispatcher-pool3-t87] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=440, retrying... 2020-05-21 17:29:40,745 TRACE [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,745 DEBUG [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,745 TRACE [RSProcedureDispatcher-pool3-t88] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,745 DEBUG [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=441 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,745 WARN [RSProcedureDispatcher-pool3-t88] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=441, retrying... 2020-05-21 17:29:40,768 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 15 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:40,768 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 15 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082190768 executing as hbase 2020-05-21 17:29:40,769 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:40,769 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:40,769 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 15 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082190768 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:40,845 TRACE [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,845 DEBUG [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,845 TRACE [RSProcedureDispatcher-pool3-t89] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,845 DEBUG [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=442 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,846 WARN [RSProcedureDispatcher-pool3-t89] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=442, retrying... 2020-05-21 17:29:40,946 TRACE [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:40,946 DEBUG [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:40,946 TRACE [RSProcedureDispatcher-pool3-t90] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:40,946 DEBUG [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=443 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:40,947 WARN [RSProcedureDispatcher-pool3-t90] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=443, retrying... 2020-05-21 17:29:41,047 TRACE [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,047 DEBUG [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,047 TRACE [RSProcedureDispatcher-pool3-t91] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,047 DEBUG [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=444 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,048 WARN [RSProcedureDispatcher-pool3-t91] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=444, retrying... 2020-05-21 17:29:41,148 TRACE [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,148 DEBUG [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,148 TRACE [RSProcedureDispatcher-pool3-t92] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,148 DEBUG [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=445 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,148 WARN [RSProcedureDispatcher-pool3-t92] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=445, retrying... 2020-05-21 17:29:41,249 TRACE [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,249 DEBUG [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,249 TRACE [RSProcedureDispatcher-pool3-t93] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,249 DEBUG [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=446 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,249 WARN [RSProcedureDispatcher-pool3-t93] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=446, retrying... 2020-05-21 17:29:41,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726656213204461 ns. 2020-05-21 17:29:41,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 90831 ns. 2020-05-21 17:29:41,349 TRACE [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,350 DEBUG [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,350 TRACE [RSProcedureDispatcher-pool3-t94] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,350 DEBUG [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=447 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,350 WARN [RSProcedureDispatcher-pool3-t94] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=447, retrying... 2020-05-21 17:29:41,450 TRACE [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,451 DEBUG [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,451 TRACE [RSProcedureDispatcher-pool3-t95] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:41,451 DEBUG [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=448 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,451 WARN [RSProcedureDispatcher-pool3-t95] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=448, retrying... 2020-05-21 17:29:41,551 TRACE [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,551 DEBUG [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,551 TRACE [RSProcedureDispatcher-pool3-t96] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,552 DEBUG [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=449 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,552 WARN [RSProcedureDispatcher-pool3-t96] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=449, retrying... 2020-05-21 17:29:41,652 TRACE [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,652 DEBUG [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,653 TRACE [RSProcedureDispatcher-pool3-t97] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:41,653 DEBUG [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=450 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,653 WARN [RSProcedureDispatcher-pool3-t97] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=450, retrying... 2020-05-21 17:29:41,753 TRACE [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,754 DEBUG [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,754 TRACE [RSProcedureDispatcher-pool3-t98] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,754 DEBUG [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=451 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,754 WARN [RSProcedureDispatcher-pool3-t98] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=451, retrying... 2020-05-21 17:29:41,854 TRACE [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,855 DEBUG [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,855 TRACE [RSProcedureDispatcher-pool3-t99] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,856 DEBUG [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=452 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,856 WARN [RSProcedureDispatcher-pool3-t99] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=452, retrying... 2020-05-21 17:29:41,956 TRACE [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:41,956 DEBUG [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:41,956 TRACE [RSProcedureDispatcher-pool3-t100] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:41,957 DEBUG [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=453 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:41,957 WARN [RSProcedureDispatcher-pool3-t100] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=453, retrying... 2020-05-21 17:29:42,057 TRACE [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,057 DEBUG [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,057 TRACE [RSProcedureDispatcher-pool3-t101] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,057 DEBUG [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=454 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,058 WARN [RSProcedureDispatcher-pool3-t101] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=454, retrying... 2020-05-21 17:29:42,142 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 541 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:42,142 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 541 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082192142 executing as hbase 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:42,143 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:42,144 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 541 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082192142 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 939 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:42,158 TRACE [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,158 DEBUG [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,158 TRACE [RSProcedureDispatcher-pool3-t102] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,158 DEBUG [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=455 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,158 WARN [RSProcedureDispatcher-pool3-t102] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=455, retrying... 2020-05-21 17:29:42,240 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 539 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:42,241 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 539 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082192241 executing as hbase 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:42,242 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 539 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082192241 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 942 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:42,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726657213218732 ns. 2020-05-21 17:29:42,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 72811 ns. 2020-05-21 17:29:42,259 TRACE [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,259 DEBUG [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,259 TRACE [RSProcedureDispatcher-pool3-t103] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,259 DEBUG [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=456 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,259 WARN [RSProcedureDispatcher-pool3-t103] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=456, retrying... 2020-05-21 17:29:42,359 TRACE [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,360 DEBUG [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,360 TRACE [RSProcedureDispatcher-pool3-t104] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,360 DEBUG [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=457 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,360 WARN [RSProcedureDispatcher-pool3-t104] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=457, retrying... 2020-05-21 17:29:42,460 TRACE [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,461 DEBUG [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,461 TRACE [RSProcedureDispatcher-pool3-t105] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,461 DEBUG [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=458 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,461 WARN [RSProcedureDispatcher-pool3-t105] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=458, retrying... 2020-05-21 17:29:42,561 TRACE [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,561 TRACE [RSProcedureDispatcher-pool3-t106] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:42,566 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:42,566 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:42,566 DEBUG [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=459 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:42,567 WARN [RSProcedureDispatcher-pool3-t106] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=459, retrying... 2020-05-21 17:29:42,667 TRACE [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,668 DEBUG [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,668 TRACE [RSProcedureDispatcher-pool3-t107] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,668 DEBUG [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=460 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,668 WARN [RSProcedureDispatcher-pool3-t107] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=460, retrying... 2020-05-21 17:29:42,770 TRACE [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,771 DEBUG [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,771 TRACE [RSProcedureDispatcher-pool3-t108] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,771 DEBUG [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=461 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,771 WARN [RSProcedureDispatcher-pool3-t108] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=461, retrying... 2020-05-21 17:29:42,872 TRACE [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,872 DEBUG [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,872 TRACE [RSProcedureDispatcher-pool3-t109] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,872 DEBUG [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=462 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,872 WARN [RSProcedureDispatcher-pool3-t109] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=462, retrying... 2020-05-21 17:29:42,973 TRACE [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:42,973 DEBUG [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:42,973 TRACE [RSProcedureDispatcher-pool3-t110] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:42,973 DEBUG [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=463 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:42,974 WARN [RSProcedureDispatcher-pool3-t110] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=463, retrying... 2020-05-21 17:29:43,074 TRACE [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,074 DEBUG [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,074 TRACE [RSProcedureDispatcher-pool3-t111] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,075 DEBUG [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=464 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,075 WARN [RSProcedureDispatcher-pool3-t111] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=464, retrying... 2020-05-21 17:29:43,175 TRACE [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,176 DEBUG [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,176 TRACE [RSProcedureDispatcher-pool3-t112] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,176 DEBUG [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=465 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,176 WARN [RSProcedureDispatcher-pool3-t112] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=465, retrying... 2020-05-21 17:29:43,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726658213258427 ns. 2020-05-21 17:29:43,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 89269 ns. 2020-05-21 17:29:43,276 TRACE [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,277 DEBUG [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,277 TRACE [RSProcedureDispatcher-pool3-t113] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,277 DEBUG [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=466 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,277 WARN [RSProcedureDispatcher-pool3-t113] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=466, retrying... 2020-05-21 17:29:43,377 TRACE [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,377 DEBUG [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,377 TRACE [RSProcedureDispatcher-pool3-t114] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,378 DEBUG [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=467 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,378 WARN [RSProcedureDispatcher-pool3-t114] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=467, retrying... 2020-05-21 17:29:43,478 TRACE [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,479 DEBUG [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,479 TRACE [RSProcedureDispatcher-pool3-t115] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,479 DEBUG [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=468 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,479 WARN [RSProcedureDispatcher-pool3-t115] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=468, retrying... 2020-05-21 17:29:43,579 TRACE [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,579 DEBUG [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,579 TRACE [RSProcedureDispatcher-pool3-t116] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,579 DEBUG [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=469 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,580 WARN [RSProcedureDispatcher-pool3-t116] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=469, retrying... 2020-05-21 17:29:43,680 TRACE [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,680 DEBUG [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,680 TRACE [RSProcedureDispatcher-pool3-t117] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,680 DEBUG [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=470 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,681 WARN [RSProcedureDispatcher-pool3-t117] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=470, retrying... 2020-05-21 17:29:43,772 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 16 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:43,772 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 16 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082193772 executing as hbase 2020-05-21 17:29:43,773 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:43,773 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:43,773 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 16 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082193772 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:43,781 TRACE [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,781 DEBUG [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,781 TRACE [RSProcedureDispatcher-pool3-t118] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,782 DEBUG [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=471 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,782 WARN [RSProcedureDispatcher-pool3-t118] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=471, retrying... 2020-05-21 17:29:43,882 TRACE [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,882 DEBUG [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,882 TRACE [RSProcedureDispatcher-pool3-t119] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,882 DEBUG [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=472 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,883 WARN [RSProcedureDispatcher-pool3-t119] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=472, retrying... 2020-05-21 17:29:43,983 TRACE [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:43,983 DEBUG [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:43,983 TRACE [RSProcedureDispatcher-pool3-t120] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:43,983 DEBUG [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=473 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:43,984 WARN [RSProcedureDispatcher-pool3-t120] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=473, retrying... 2020-05-21 17:29:44,084 TRACE [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,084 DEBUG [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,084 TRACE [RSProcedureDispatcher-pool3-t121] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,085 DEBUG [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=474 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,085 WARN [RSProcedureDispatcher-pool3-t121] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=474, retrying... 2020-05-21 17:29:44,185 TRACE [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,185 DEBUG [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,185 TRACE [RSProcedureDispatcher-pool3-t122] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,197 DEBUG [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=475 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,197 WARN [RSProcedureDispatcher-pool3-t122] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=475, retrying... 2020-05-21 17:29:44,245 TRACE [ReadOnlyZKClient-zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181@0x1054097a] zookeeper.ReadOnlyZKClient: 0x1054097a to zookeeper-0-zookeeper.zookeeper:2181,zookeeper-1-zookeeper.zookeeper:2181,zookeeper-2-zookeeper.zookeeper:2181 inactive for 60000ms; closing (Will reconnect when new requests) 2020-05-21 17:29:44,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726659213262875 ns. 2020-05-21 17:29:44,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 69242 ns. 2020-05-21 17:29:44,298 TRACE [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,298 DEBUG [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,298 TRACE [RSProcedureDispatcher-pool3-t123] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,298 DEBUG [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=476 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,298 WARN [RSProcedureDispatcher-pool3-t123] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=476, retrying... 2020-05-21 17:29:44,383 TRACE [ProcExecTimeout] procedure2.TimeoutExecutorThread: Executing containedObject=pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore, timeout=1590082184382, delay=0 2020-05-21 17:29:44,399 TRACE [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,399 DEBUG [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,399 TRACE [RSProcedureDispatcher-pool3-t124] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,399 DEBUG [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=477 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,399 WARN [RSProcedureDispatcher-pool3-t124] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=477, retrying... 2020-05-21 17:29:44,499 TRACE [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,500 DEBUG [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,500 TRACE [RSProcedureDispatcher-pool3-t125] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,500 DEBUG [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=478 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,500 WARN [RSProcedureDispatcher-pool3-t125] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=478, retrying... 2020-05-21 17:29:44,578 TRACE [WorkerMonitor] procedure2.TimeoutExecutorThread: Executing timeout=1590082184578, delay=0 2020-05-21 17:29:44,600 TRACE [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,601 TRACE [RSProcedureDispatcher-pool3-t126] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:44,605 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:44,605 DEBUG [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=479 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:44,605 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:44,606 WARN [RSProcedureDispatcher-pool3-t126] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=479, retrying... 2020-05-21 17:29:44,706 TRACE [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,707 DEBUG [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,707 TRACE [RSProcedureDispatcher-pool3-t127] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,707 DEBUG [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=480 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,707 WARN [RSProcedureDispatcher-pool3-t127] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=480, retrying... 2020-05-21 17:29:44,807 TRACE [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,808 DEBUG [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,808 TRACE [RSProcedureDispatcher-pool3-t128] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,808 DEBUG [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=481 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,809 WARN [RSProcedureDispatcher-pool3-t128] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=481, retrying... 2020-05-21 17:29:44,909 TRACE [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:44,910 DEBUG [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:44,910 TRACE [RSProcedureDispatcher-pool3-t129] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:44,910 DEBUG [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=482 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:44,911 WARN [RSProcedureDispatcher-pool3-t129] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=482, retrying... 2020-05-21 17:29:45,011 TRACE [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,011 DEBUG [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,012 TRACE [RSProcedureDispatcher-pool3-t2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:45,012 DEBUG [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=483 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,013 WARN [RSProcedureDispatcher-pool3-t2] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=483, retrying... 2020-05-21 17:29:45,113 TRACE [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,113 DEBUG [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,113 TRACE [RSProcedureDispatcher-pool3-t3] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,113 DEBUG [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=484 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,113 WARN [RSProcedureDispatcher-pool3-t3] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=484, retrying... 2020-05-21 17:29:45,146 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 542 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:45,146 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 542 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082195146 executing as hbase 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:45,147 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:45,148 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 542 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082195146 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 996 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:45,214 TRACE [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,214 DEBUG [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,214 TRACE [RSProcedureDispatcher-pool3-t4] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,214 DEBUG [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=485 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,215 WARN [RSProcedureDispatcher-pool3-t4] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=485, retrying... 2020-05-21 17:29:45,245 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 540 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:45,245 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 540 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082195245 executing as hbase 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:45,246 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 540 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082195245 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 975 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:45,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726660213263568 ns. 2020-05-21 17:29:45,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 191082 ns. 2020-05-21 17:29:45,315 TRACE [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,315 DEBUG [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,315 TRACE [RSProcedureDispatcher-pool3-t5] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,316 DEBUG [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=486 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,316 WARN [RSProcedureDispatcher-pool3-t5] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=486, retrying... 2020-05-21 17:29:45,416 TRACE [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,417 DEBUG [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,417 TRACE [RSProcedureDispatcher-pool3-t6] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,417 DEBUG [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=487 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,418 WARN [RSProcedureDispatcher-pool3-t6] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=487, retrying... 2020-05-21 17:29:45,518 TRACE [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,518 DEBUG [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,518 TRACE [RSProcedureDispatcher-pool3-t7] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,518 DEBUG [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=488 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,519 WARN [RSProcedureDispatcher-pool3-t7] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=488, retrying... 2020-05-21 17:29:45,619 TRACE [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,619 DEBUG [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,619 TRACE [RSProcedureDispatcher-pool3-t8] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,619 DEBUG [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=489 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,619 WARN [RSProcedureDispatcher-pool3-t8] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=489, retrying... 2020-05-21 17:29:45,719 TRACE [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,719 DEBUG [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,719 TRACE [RSProcedureDispatcher-pool3-t9] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,720 DEBUG [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=490 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,720 WARN [RSProcedureDispatcher-pool3-t9] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=490, retrying... 2020-05-21 17:29:45,820 TRACE [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,821 DEBUG [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,821 TRACE [RSProcedureDispatcher-pool3-t10] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:45,821 DEBUG [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=491 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,822 WARN [RSProcedureDispatcher-pool3-t10] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=491, retrying... 2020-05-21 17:29:45,922 TRACE [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:45,922 DEBUG [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:45,922 TRACE [RSProcedureDispatcher-pool3-t11] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:45,922 DEBUG [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=492 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:45,922 WARN [RSProcedureDispatcher-pool3-t11] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=492, retrying... 2020-05-21 17:29:46,023 TRACE [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,023 DEBUG [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,023 TRACE [RSProcedureDispatcher-pool3-t13] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,023 DEBUG [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=493 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,023 WARN [RSProcedureDispatcher-pool3-t13] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=493, retrying... 2020-05-21 17:29:46,124 TRACE [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,124 DEBUG [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,124 TRACE [RSProcedureDispatcher-pool3-t12] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,124 DEBUG [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=494 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,125 WARN [RSProcedureDispatcher-pool3-t12] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=494, retrying... 2020-05-21 17:29:46,225 TRACE [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,225 DEBUG [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,225 TRACE [RSProcedureDispatcher-pool3-t14] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,226 DEBUG [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=495 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,226 WARN [RSProcedureDispatcher-pool3-t14] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=495, retrying... 2020-05-21 17:29:46,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726661213256831 ns. 2020-05-21 17:29:46,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 88159 ns. 2020-05-21 17:29:46,327 TRACE [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,327 DEBUG [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,327 TRACE [RSProcedureDispatcher-pool3-t15] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,327 DEBUG [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=496 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,328 WARN [RSProcedureDispatcher-pool3-t15] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=496, retrying... 2020-05-21 17:29:46,428 TRACE [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,428 DEBUG [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,428 TRACE [RSProcedureDispatcher-pool3-t16] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,428 DEBUG [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=497 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,429 WARN [RSProcedureDispatcher-pool3-t16] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=497, retrying... 2020-05-21 17:29:46,529 TRACE [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,529 DEBUG [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,529 TRACE [RSProcedureDispatcher-pool3-t17] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,529 DEBUG [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=498 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,531 WARN [RSProcedureDispatcher-pool3-t17] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=498, retrying... 2020-05-21 17:29:46,631 TRACE [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,631 TRACE [RSProcedureDispatcher-pool3-t18] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:46,636 TRACE [RS-EventLoopGroup-1-2] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:46,637 DEBUG [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=499 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:46,637 DEBUG [RS-EventLoopGroup-1-2] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:46,637 WARN [RSProcedureDispatcher-pool3-t18] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=499, retrying... 2020-05-21 17:29:46,738 TRACE [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,738 DEBUG [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,738 TRACE [RSProcedureDispatcher-pool3-t19] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,738 DEBUG [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=500 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,738 WARN [RSProcedureDispatcher-pool3-t19] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=500, retrying... 2020-05-21 17:29:46,777 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 17 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 130 bytes 2020-05-21 17:29:46,777 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 17 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082196777 executing as hbase 2020-05-21 17:29:46,777 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: ReportOnlineRegions regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 regionCount=0, metaLoaded=true [] 2020-05-21 17:29:46,777 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: no online region found on regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 2020-05-21 17:29:46,777 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 17 service: RegionServerStatusService methodName: RegionServerReport size: 130 connection: 10.128.14.40:39393 deadline: 1590082196777 param: server host_name: "regionserver-2.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590082132059 load { numberOfRequests: 0 } connection: 10.128.14.40:39393, response queueTime: 0 processingTime: 0 totalTime: 0 2020-05-21 17:29:46,838 TRACE [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,839 DEBUG [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,839 TRACE [RSProcedureDispatcher-pool3-t20] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,839 DEBUG [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=501 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,839 WARN [RSProcedureDispatcher-pool3-t20] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=501, retrying... 2020-05-21 17:29:46,939 TRACE [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:46,939 DEBUG [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:46,939 TRACE [RSProcedureDispatcher-pool3-t21] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:46,940 DEBUG [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=502 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:46,940 WARN [RSProcedureDispatcher-pool3-t21] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=502, retrying... 2020-05-21 17:29:47,040 TRACE [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,040 DEBUG [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,040 TRACE [RSProcedureDispatcher-pool3-t22] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,040 DEBUG [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=503 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,041 WARN [RSProcedureDispatcher-pool3-t22] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=503, retrying... 2020-05-21 17:29:47,051 TRACE [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: doRun called: {} 2020-05-21 17:29:47,141 TRACE [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,141 DEBUG [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,141 TRACE [RSProcedureDispatcher-pool3-t23] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,142 DEBUG [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=504 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,142 WARN [RSProcedureDispatcher-pool3-t23] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=504, retrying... 2020-05-21 17:29:47,242 TRACE [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,242 DEBUG [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,242 TRACE [RSProcedureDispatcher-pool3-t24] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,242 DEBUG [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=505 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,243 WARN [RSProcedureDispatcher-pool3-t24] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=505, retrying... 2020-05-21 17:29:47,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726662213251492 ns. 2020-05-21 17:29:47,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 96354 ns. 2020-05-21 17:29:47,343 TRACE [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,343 DEBUG [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,343 TRACE [RSProcedureDispatcher-pool3-t25] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,343 DEBUG [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=506 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,344 WARN [RSProcedureDispatcher-pool3-t25] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=506, retrying... 2020-05-21 17:29:47,444 TRACE [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,444 DEBUG [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,444 TRACE [RSProcedureDispatcher-pool3-t26] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,444 DEBUG [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=507 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,444 WARN [RSProcedureDispatcher-pool3-t26] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=507, retrying... 2020-05-21 17:29:47,545 TRACE [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,545 DEBUG [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,545 TRACE [RSProcedureDispatcher-pool3-t27] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,545 DEBUG [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=508 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,545 WARN [RSProcedureDispatcher-pool3-t27] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=508, retrying... 2020-05-21 17:29:47,646 TRACE [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,646 DEBUG [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,646 TRACE [RSProcedureDispatcher-pool3-t28] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,646 DEBUG [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=509 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,646 WARN [RSProcedureDispatcher-pool3-t28] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=509, retrying... 2020-05-21 17:29:47,746 TRACE [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,747 DEBUG [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,747 TRACE [RSProcedureDispatcher-pool3-t29] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,747 DEBUG [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=510 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,748 WARN [RSProcedureDispatcher-pool3-t29] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=510, retrying... 2020-05-21 17:29:47,848 TRACE [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,848 DEBUG [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,848 TRACE [RSProcedureDispatcher-pool3-t30] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,848 DEBUG [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=511 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,848 WARN [RSProcedureDispatcher-pool3-t30] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=511, retrying... 2020-05-21 17:29:47,949 TRACE [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:47,949 DEBUG [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:47,949 TRACE [RSProcedureDispatcher-pool3-t31] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:47,949 DEBUG [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=512 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:47,950 WARN [RSProcedureDispatcher-pool3-t31] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=512, retrying... 2020-05-21 17:29:48,050 TRACE [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,050 DEBUG [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,050 TRACE [RSProcedureDispatcher-pool3-t32] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,050 DEBUG [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=513 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,050 WARN [RSProcedureDispatcher-pool3-t32] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=513, retrying... 2020-05-21 17:29:48,150 TRACE [RS-EventLoopGroup-1-1] ipc.RpcServer: RequestHeader call_id: 543 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4077 bytes 2020-05-21 17:29:48,151 TRACE [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,151 DEBUG [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,151 TRACE [RSProcedureDispatcher-pool3-t33] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,151 DEBUG [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=514 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,151 WARN [RSProcedureDispatcher-pool3-t33] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=514, retrying... 2020-05-21 17:29:48,151 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 543 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082198151 executing as hbase 2020-05-21 17:29:48,152 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 82ccc4215fd1fe7a2a329a0c6c297fa4, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6bc10ee35b217018a9e0d67385305e9, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a6d29d67bde8e42b9fe72ebb4df3be68, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9eda3f5973caeeba814cec96879ff797, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 895c51e25b71d8edc8d063478d5e959d, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 7bc9b31cffc00199e00820c96bb73d39, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: fb8afdaeaa529390bffddea5ea7a76fd, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 85da959eab50f93627ef35794d04ba36, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1f39c96ff0915a836361d227823976f7, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 6b3928169202b04437a412abcc3c3e84, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 8a83dfc2fd90e7fc0f6ee8c530093fa2, family=e, existingValue=142840890, completeSequenceId=142840890 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 10c81f35e28121d9c1f3f730c673728b, family=e, existingValue=98020795, completeSequenceId=98020795 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 2d70a3e092fcb7c31e732610f34e42cb, family=e, existingValue=119573495, completeSequenceId=119573495 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 9bf8f6c9ac2972f06898c86000e5f79f, family=e, existingValue=109978746, completeSequenceId=109978746 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 86533e2f7328258a6b5448a4b09aee15, family=e, existingValue=109023215, completeSequenceId=109023215 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: f4e57ea829d51cdcb1c8d74a938311c2, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: bcfcfdf8cf1e7461eb7353580e638d1a, family=e, existingValue=104264130, completeSequenceId=104264130 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: e855ef99e11ec78bead3c9e21e6da12f, family=e, existingValue=90604375, completeSequenceId=90604375 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 5b21c77606219c0cd30298e8c40566fa, family=e, existingValue=105781306, completeSequenceId=105781306 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 595f47d1f2db471835be0c1ce74b91f5, family=e, existingValue=111331359, completeSequenceId=111331359 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: a20cc6185462b4c4f9e590d517ad8746, family=e, existingValue=94045342, completeSequenceId=94045342 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 66abcacc5fe6701f54279e5cc22b758f, family=e, existingValue=98434719, completeSequenceId=98434719 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=table, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=info, existingValue=25638, completeSequenceId=25638 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] master.ServerManager: 1588230740, family=rep_barrier, existingValue=25635, completeSequenceId=25635 2020-05-21 17:29:48,153 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] assignment.AssignmentManager: 33e2f7328258a6b5448a4b09aee15., aeris_v2,\x09\xCD\x027\x97\xC4\x05Sysdb\xC4\x0Benvironment\xC4\x06archer\xC4\x0Btemperature\xC4\x06status\xC4\x06system\xC4\x0CTempSensor24,1589511922840.f4e57ea829d51cdcb1c8d74a938311c2., aeris_v2,\x09\xCD\x02\x87\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1589511922840.bcfcfdf8cf1e7461eb7353580e638d1a., aeris_v2,\x0F,1587075439265.e855ef99e11ec78bead3c9e21e6da12f., aeris_v2,\x0F\xCD\x02I\x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587075440061.5b21c77606219c0cd30298e8c40566fa., aeris_v2,\x10\xCD\x02 \x96\xC4\x05Smash\xC4\x08counters\xC4\x07ethIntf\xC4\x0EStrataCounters\xC4\x07current\xC4\x07counter,1587068227635.595f47d1f2db471835be0c1ce74b91f5., aeris_v2,\x11\xCD\x02 \x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0211,1589394141211.a20cc6185462b4c4f9e590d517ad8746., aeris_v2,,1587075425654.66abcacc5fe6701f54279e5cc22b758f., hbase:meta,,1] 2020-05-21 17:29:48,154 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=16000] ipc.RpcServer: callId: 543 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.9.37:50537 deadline: 1590082198151 param: server host_name: "regionserver-1.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081081754 load { numberOfRequests: 614 } connection: 10.128.9.37:50537, response queueTime: 0 processingTime: 3 totalTime: 3 2020-05-21 17:29:48,249 TRACE [RS-EventLoopGroup-1-2] ipc.RpcServer: RequestHeader call_id: 541 method_name: "RegionServerReport" request_param: true priority: 0 timeout: 10000 totalRequestSize: 4136 bytes 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 541 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082198249 executing as hbase 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: ba13fae1aa340cfe87b198638663432f, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6c4c2be586f7ad04823bf64879bf863e, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 5a90663185105f199068aadf7f495f28, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 47a8ba48785bcfd5a3bf576c212eb570, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 521214e7702689731813492a1dc4b54a, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6fc9d3397f8a7edbd4dd190aaef975c3, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 9edb5377034d812a87d2487858346194, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,249 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 45b9358748dd408f57d5ee0a533f5934, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 20404052b3c0674a71c9f9da299b9e17, family=e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6d850acf1457152a77813dc3c02d3d86, family=e, existingValue=116813585, completeSequenceId=116813585 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: eb1b85293f32138e11c1885be335155b, family=e, existingValue=113655440, completeSequenceId=113655440 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fbbd176edeb2884b2cab67dbf033f887, family=e, existingValue=113014537, completeSequenceId=113014537 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 94642645d0ba481c1c06c18a7b31f122, family=e, existingValue=98018524, completeSequenceId=98018524 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 4d52816856fdb4e2a80d432d24b02872, family=e, existingValue=115582491, completeSequenceId=115582491 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 789e37c023bf4b11823c10883a42ef94, family=e, existingValue=115233623, completeSequenceId=115233623 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, existingValue=121135426, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 707360501b0ab41efd74025af857bbb7, family=e, existingValue=121135431, completeSequenceId=121135431 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 6247174694aa0f08b4248bcfcb7e8b9e, family=e, existingValue=98085642, completeSequenceId=98085642 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: fd76870dec2ac0f1e1f8fe0f563b5611, family=e, existingValue=100510637, completeSequenceId=100510637 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: cb592da589b3dd3af1fd129c458e2404, family=e, existingValue=80144966, completeSequenceId=80144966 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 92440ed8ef80bb8614befd3e89e39326, family=e, existingValue=103551155, completeSequenceId=103551155 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 58c894e046698664c5c4b16292cf95ee, family=e, existingValue=100968486, completeSequenceId=100968486 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] master.ServerManager: 868e9d680ced53e3a97e0ff466d6aef9, family=info, existingValue=-1, completeSequenceId=-1 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] assignment.AssignmentManager: current\xC4\x07counter,1587075436855.707360501b0ab41efd74025af857bbb7., aeris_v2,\x0D\xCD\x025\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x041642,1587075438155.6247174694aa0f08b4248bcfcb7e8b9e., aeris_v2,\x0E&\x94\xC4\x06Kernel\xC4\x04proc\xC4\x04stat\xC4\x0525402,1587075438822.fd76870dec2ac0f1e1f8fe0f563b5611., aeris_v2,\x12\x08\x97\xC4\x07Devices\xC4\x0BHSH14420380\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet14\xC4\x05rates,1589394141211.cb592da589b3dd3af1fd129c458e2404., aeris_v2,\x12\x08\x99\xC4\x07Devices\xC4\x0BJPE17082089\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x0AEthernet39\xC4\x09aggregate\xC4\x05rates\xC4\x021m,1587075442855.92440ed8ef80bb8614befd3e89e39326., aeris_v2,\x13\x08\x99\xC4\x07Devices\xC4\x0BSSJ18431670\xC4\x0Eversioned-data\xC4\x0Ainterfaces\xC4\x04data\xC4\x09Ethernet2\xC4\x09aggregate\xC4\x05rates\xC4\x0315m,1587075445857.58c894e046698664c5c4b16292cf95ee., hbase:namespace,,1581721508192.868e9d680ced53e3a97e0ff466d6aef9.] 2020-05-21 17:29:48,250 TRACE [RpcServer.priority.RWQ.Fifo.write.handler=1,queue=0,port=16000] ipc.RpcServer: callId: 541 service: RegionServerStatusService methodName: RegionServerReport size: 4.0 K connection: 10.128.53.53:44503 deadline: 1590082198249 param: server host_name: "regionserver-0.hbase.hbase.svc.cluster.local" port: 16020 start_code: 1590081082066 load { numberOfRequests: 617 } connection: 10.128.53.53:44503, response queueTime: 0 processingTime: 1 totalTime: 1 2020-05-21 17:29:48,252 TRACE [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,252 DEBUG [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,252 TRACE [RSProcedureDispatcher-pool3-t34] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,252 DEBUG [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=515 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,252 WARN [RSProcedureDispatcher-pool3-t34] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=515, retrying... 2020-05-21 17:29:48,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726663213179691 ns. 2020-05-21 17:29:48,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 97262 ns. 2020-05-21 17:29:48,353 TRACE [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,353 DEBUG [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,353 TRACE [RSProcedureDispatcher-pool3-t35] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,353 DEBUG [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=516 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,353 WARN [RSProcedureDispatcher-pool3-t35] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=516, retrying... 2020-05-21 17:29:48,453 TRACE [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,454 DEBUG [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,454 TRACE [RSProcedureDispatcher-pool3-t36] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,454 DEBUG [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=517 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,455 WARN [RSProcedureDispatcher-pool3-t36] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=517, retrying... 2020-05-21 17:29:48,555 TRACE [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,555 DEBUG [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,555 TRACE [RSProcedureDispatcher-pool3-t37] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,555 DEBUG [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=518 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,556 WARN [RSProcedureDispatcher-pool3-t37] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=518, retrying... 2020-05-21 17:29:48,656 TRACE [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,656 TRACE [RSProcedureDispatcher-pool3-t38] ipc.NettyRpcConnection: Connecting to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:48,661 TRACE [RS-EventLoopGroup-1-1] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 5ms 2020-05-21 17:29:48,661 DEBUG [RS-EventLoopGroup-1-1] ipc.FailedServers: Added failed server with address regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 2020-05-21 17:29:48,661 DEBUG [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=519 java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:177) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:92) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:304) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1426) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:326) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:312) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:924) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:179) at org.apache.hadoop.hbase.ipc.NettyRpcConnection.access$500(NettyRpcConnection.java:71) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:267) at org.apache.hadoop.hbase.ipc.NettyRpcConnection$3.operationComplete(NettyRpcConnection.java:261) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:502) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:495) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:474) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:415) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:540) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:533) at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:114) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:631) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:650) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:524) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417) at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:328) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.throwConnectException(Errors.java:112) at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:269) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:667) at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:644) ... 6 more Caused by: java.net.ConnectException: finishConnect(..) failed: No route to host ... 10 more 2020-05-21 17:29:48,662 WARN [RSProcedureDispatcher-pool3-t38] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to java.net.ConnectException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: No route to host: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=519, retrying... 2020-05-21 17:29:48,763 TRACE [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,763 DEBUG [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,763 TRACE [RSProcedureDispatcher-pool3-t39] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,763 DEBUG [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=520 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,763 WARN [RSProcedureDispatcher-pool3-t39] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=520, retrying... 2020-05-21 17:29:48,864 TRACE [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,864 DEBUG [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,864 TRACE [RSProcedureDispatcher-pool3-t40] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,864 DEBUG [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=521 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,864 WARN [RSProcedureDispatcher-pool3-t40] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=521, retrying... 2020-05-21 17:29:48,964 TRACE [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:48,965 DEBUG [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:48,965 TRACE [RSProcedureDispatcher-pool3-t41] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:48,965 DEBUG [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=522 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:48,965 WARN [RSProcedureDispatcher-pool3-t41] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=522, retrying... 2020-05-21 17:29:49,066 TRACE [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:49,066 DEBUG [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:49,066 TRACE [RSProcedureDispatcher-pool3-t42] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:49,066 DEBUG [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=523 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:49,066 WARN [RSProcedureDispatcher-pool3-t42] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=523, retrying... 2020-05-21 17:29:49,166 TRACE [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:49,167 DEBUG [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:49,167 TRACE [RSProcedureDispatcher-pool3-t43] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:49,167 DEBUG [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=524 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:49,167 WARN [RSProcedureDispatcher-pool3-t43] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=524, retrying... 2020-05-21 17:29:49,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - start to measure at: 1726664213172067 ns. 2020-05-21 17:29:49,256 TRACE [master/hbasemaster-0:16000.splitLogManager..Chore.1] util.MovingAverage: SplitLogManager Timeout Monitor - elapse: 91377 ns. 2020-05-21 17:29:49,267 TRACE [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:49,268 DEBUG [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:49,268 TRACE [RSProcedureDispatcher-pool3-t44] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 0ms 2020-05-21 17:29:49,268 DEBUG [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=525 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:49,268 WARN [RSProcedureDispatcher-pool3-t44] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=525, retrying... 2020-05-21 17:29:49,368 TRACE [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: Building request with operations count=1 2020-05-21 17:29:49,369 DEBUG [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Not trying to connect to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 this server is in the failed servers list 2020-05-21 17:29:49,369 TRACE [RSProcedureDispatcher-pool3-t45] ipc.AbstractRpcClient: Call: ExecuteProcedures, callTime: 1ms 2020-05-21 17:29:49,369 DEBUG [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed, try=526 org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:423) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:419) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:436) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:330) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$200(AbstractRpcClient.java:97) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:585) at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$BlockingStub.executeProcedures(AdminProtos.java:31006) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:349) at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:314) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:354) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:433) ... 9 more 2020-05-21 17:29:49,369 WARN [RSProcedureDispatcher-pool3-t45] procedure.RSProcedureDispatcher: request to server regionserver-2.hbase.hbase.svc.cluster.local,16020,1590082132059 failed due to org.apache.hadoop.hbase.ipc.FailedServerException: Call to regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: regionserver-2.hbase.hbase.svc.cluster.local/10.128.14.39:16020, try=526, retrying...