Mon Sep 18 14:17:18 UTC 2023 Starting master on ee4a85363fe2 core file size (blocks, -c) unlimited data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 768035 max locked memory (kbytes, -l) unlimited max memory size (kbytes, -m) unlimited open files (-n) 100000 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 32768 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited 2023-09-18 14:17:23,285 INFO [main] master.HMaster: STARTING service HMaster 2023-09-18 14:17:23,294 INFO [main] util.VersionInfo: HBase 2.4.17 2023-09-18 14:17:23,294 INFO [main] util.VersionInfo: Source code repository git://e0b8ecc3178f/home/taklwu/hbase-rm/output/hbase revision=7fd096f39b4284da9a71da3ce67c48d259ffa79a 2023-09-18 14:17:23,294 INFO [main] util.VersionInfo: Compiled by taklwu on Fri Mar 31 18:10:45 UTC 2023 2023-09-18 14:17:23,294 INFO [main] util.VersionInfo: From source with checksum 0e34884c8e1d6e46ba560b6a824d684ba4dd42b8f34a4318e57510627a88e3e41e34bdb8f06da56a46d8065f331aa349ad58755cf6db5d75a5742f4174ccf81d 2023-09-18 14:17:23,747 INFO [main] util.ServerCommandLine: hbase.tmp.dir: ./tmp 2023-09-18 14:17:23,747 INFO [main] util.ServerCommandLine: hbase.rootdir: hdfs://master:8020/hbase 2023-09-18 14:17:23,747 INFO [main] util.ServerCommandLine: hbase.cluster.distributed: true 2023-09-18 14:17:23,747 INFO [main] util.ServerCommandLine: hbase.zookeeper.quorum: hmaster,hregion1,hregion2 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:HBASE_LOGFILE=hbase--master-ee4a85363fe2.log 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:SUPERVISOR_ENABLED=1 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:IS_HMASTER=true 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:PYTHON=python2 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:HBASE_CONF_DIR=/etc/hbase-2.4.17 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:HBASE_SHELL_DAEMON_PORT=36000 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:CUR_STATUS=ORI 2023-09-18 14:17:23,758 INFO [main] util.ServerCommandLine: env:HBASE_SHELL_HOST=192.168.243.2 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:JAVA_TOOL_OPTIONS=-javaagent:/org.jacoco.agent.rt.jar=append=false,includes=org.apache.hadoop.hbase.*,excludes=org.jruby.*,output=dfe,address=192.168.243.1,port=38671,sessionid=hbase-O3JUt7cD_original-0 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_REST_OPTS= 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:JVM_PID=587 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:PWD=/ 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HADOOP_IP=192.168.243.102 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_SEEDS=192.168.243.2, 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_ROOT_LOGGER=INFO,RFA 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_ENV_INIT=true 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_IDENT_STRING= 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_ZNODE_FILE=/tmp/hbase--master.znode 2023-09-18 14:17:23,759 INFO [main] util.ServerCommandLine: env:HBASE_LOG_DIR=/var/log/hbase 2023-09-18 14:17:23,760 INFO [main] util.ServerCommandLine: env:HBASE_LOG_PREFIX=hbase--master-ee4a85363fe2 2023-09-18 14:17:23,761 INFO [main] util.ServerCommandLine: ose-jwt-4.41.1.jar:/hbase/hbase-2.4.17/lib/objenesis-2.6.jar:/hbase/hbase-2.4.17/lib/okhttp-2.7.5.jar:/hbase/hbase-2.4.17/lib/okio-1.6.0.jar:/hbase/hbase-2.4.17/lib/paranamer-2.3.jar:/hbase/hbase-2.4.17/lib/protobuf-java-2.5.0.jar:/hbase/hbase-2.4.17/lib/snappy-java-1.0.5.jar:/hbase/hbase-2.4.17/lib/spymemcached-2.12.2.jar:/hbase/hbase-2.4.17/lib/stax2-api-3.1.4.jar:/hbase/hbase-2.4.17/lib/woodstox-core-5.0.3.jar:/hbase/hbase-2.4.17/lib/xmlenc-0.52.jar:/hbase/hbase-2.4.17/lib/zookeeper-3.5.7.jar:/hbase/hbase-2.4.17/lib/zookeeper-jute-3.5.7.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/audience-annotations-0.13.0.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/commons-logging-1.2.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/reload4j-1.2.22.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-api-1.7.33.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:SUPERVISOR_SERVER_URL=unix:///var/run/supervisor.sock 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:HBASE_CLUSTER_NAME=dev_cluster 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:HBASE_LOGGING_LEVEL=DEBUG 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:HBASE_AUTOSTART_FILE=/tmp/hbase--master.autostart 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:SED=sed 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:HOSTNAME=ee4a85363fe2 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:SUPERVISOR_GROUP_NAME=upfuzz_hbase 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:SUPERVISOR_PROCESS_NAME=hbase 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:GREP=grep 2023-09-18 14:17:23,762 INFO [main] util.ServerCommandLine: env:HBASE_NICENESS=0 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HBASE_SECURITY_LOGGER=INFO,RFAS 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HBASE_OPTS=-XX:+UseConcMarkSweepGC -Djava.util.logging.config.class=org.apache.hadoop.hbase.logging.JulToSlf4jInitializer -Dhbase.log.dir=/var/log/hbase -Dhbase.log.file=hbase--master-ee4a85363fe2.log -Dhbase.home.dir=/hbase/hbase-2.4.17 -Dhbase.id.str= -Dhbase.root.logger=INFO,RFA -Dhbase.security.logger=INFO,RFAS 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HBASE_HOME=/hbase/hbase-2.4.17 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HBASE_THRIFT_OPTS= 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HBASE_CONF=/etc/hbase-2.4.17 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:SHLVL=5 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:HOME=/root 2023-09-18 14:17:23,763 INFO [main] util.ServerCommandLine: env:MALLOC_ARENA_MAX=4 2023-09-18 14:17:23,765 INFO [main] util.ServerCommandLine: vmName=OpenJDK 64-Bit Server VM, vmVendor=Private Build, vmVersion=25.362-b09 2023-09-18 14:17:23,766 INFO [main] util.ServerCommandLine: vmInputArguments=[-javaagent:/org.jacoco.agent.rt.jar=append=false,includes=org.apache.hadoop.hbase.*,excludes=org.jruby.*,output=dfe,address=192.168.243.1,port=38671,sessionid=hbase-O3JUt7cD_original-0, -Dproc_master, -XX:OnOutOfMemoryError=kill -9 %p, -XX:+UseConcMarkSweepGC, -Djava.util.logging.config.class=org.apache.hadoop.hbase.logging.JulToSlf4jInitializer, -Dhbase.log.dir=/var/log/hbase, -Dhbase.log.file=hbase--master-ee4a85363fe2.log, -Dhbase.home.dir=/hbase/hbase-2.4.17, -Dhbase.id.str=, -Dhbase.root.logger=INFO,RFA, -Dhbase.security.logger=INFO,RFAS] 2023-09-18 14:17:24,754 INFO [main] metrics.MetricRegistries: Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-09-18 14:17:24,845 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-09-18 14:17:27,008 INFO [main] regionserver.RSRpcServices: master/hmaster:16000 server-side Connection retries=45 2023-09-18 14:17:27,111 INFO [main] ipc.RpcExecutor: Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=3, maxQueueLength=300, handlerCount=30 2023-09-18 14:17:27,118 INFO [main] ipc.RpcExecutor: Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=300, handlerCount=20 2023-09-18 14:17:27,118 INFO [main] ipc.RWQueueRpcExecutor: priority.RWQ.Fifo writeQueues=1 writeHandlers=2 readQueues=1 readHandlers=18 scanQueues=0 scanHandlers=0 2023-09-18 14:17:27,118 INFO [main] ipc.RpcExecutor: Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=300, handlerCount=3 2023-09-18 14:17:27,119 INFO [main] ipc.RpcExecutor: Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=300, handlerCount=1 2023-09-18 14:17:27,626 INFO [main] ipc.RpcServerFactory: Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService 2023-09-18 14:17:27,728 INFO [main] Configuration.deprecation: hbase.ipc.server.reservoir.initial.buffer.size is deprecated. Instead, use hbase.server.allocator.buffer.size 2023-09-18 14:17:27,729 INFO [main] Configuration.deprecation: hbase.ipc.server.reservoir.initial.max is deprecated. Instead, use hbase.server.allocator.max.buffer.count 2023-09-18 14:17:27,966 INFO [main] ipc.NettyRpcServer: Bind to /192.168.243.2:16000 2023-09-18 14:17:28,982 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-18 14:17:28,989 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-18 14:17:29,040 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=master:16000 connecting to ZooKeeper ensemble=hmaster:2181,hregion1:2181,hregion2:2181 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.5.7-f0fdd52973d373ffd9c86b81d99842dc2c7f660e, built on 02/10/2020 11:30 GMT 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=hmaster 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.8.0_362 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Private Build 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre 2023-09-18 14:17:29,049 INFO [main] zookeeper.ZooKeeper: /hbase-2.4.17/lib/objenesis-2.6.jar:/hbase/hbase-2.4.17/lib/okhttp-2.7.5.jar:/hbase/hbase-2.4.17/lib/okio-1.6.0.jar:/hbase/hbase-2.4.17/lib/paranamer-2.3.jar:/hbase/hbase-2.4.17/lib/protobuf-java-2.5.0.jar:/hbase/hbase-2.4.17/lib/snappy-java-1.0.5.jar:/hbase/hbase-2.4.17/lib/spymemcached-2.12.2.jar:/hbase/hbase-2.4.17/lib/stax2-api-3.1.4.jar:/hbase/hbase-2.4.17/lib/woodstox-core-5.0.3.jar:/hbase/hbase-2.4.17/lib/xmlenc-0.52.jar:/hbase/hbase-2.4.17/lib/zookeeper-3.5.7.jar:/hbase/hbase-2.4.17/lib/zookeeper-jute-3.5.7.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/audience-annotations-0.13.0.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/commons-logging-1.2.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/reload4j-1.2.22.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-api-1.7.33.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar:/org.jacoco.agent.rt.jar 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=6.2.0-26-generic 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/ 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.free=1789MB 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.max=30485MB 2023-09-18 14:17:29,050 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.total=1979MB 2023-09-18 14:17:29,053 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hmaster:2181,hregion1:2181,hregion2:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@4e885b9 2023-09-18 14:17:29,059 INFO [main] common.X509Util: Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation 2023-09-18 14:17:29,064 INFO [main] zookeeper.ClientCnxnSocket: jute.maxbuffer value is 4194304 Bytes 2023-09-18 14:17:29,075 INFO [main] zookeeper.ClientCnxn: zookeeper.request.timeout value is 0. feature enabled= 2023-09-18 14:17:29,088 INFO [main-SendThread(hregion1:2181)] zookeeper.ClientCnxn: Opening socket connection to server hregion1/192.168.243.3:2181. Will not attempt to authenticate using SASL (unknown error) 2023-09-18 14:17:29,096 INFO [main-SendThread(hregion1:2181)] zookeeper.ClientCnxn: Socket connection established, initiating session, client: /192.168.243.2:50986, server: hregion1/192.168.243.3:2181 2023-09-18 14:17:29,116 INFO [main-SendThread(hregion1:2181)] zookeeper.ClientCnxn: Session establishment complete on server hregion1/192.168.243.3:2181, sessionid = 0x10050be756c0001, negotiated timeout = 90000 2023-09-18 14:17:29,407 INFO [main] util.log: Logging initialized @9651ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-09-18 14:17:29,604 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-09-18 14:17:29,605 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-09-18 14:17:29,607 INFO [main] http.HttpServer: Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-09-18 14:17:29,610 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-09-18 14:17:29,610 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-09-18 14:17:29,610 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-09-18 14:17:29,626 INFO [main] http.HttpServer: ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-09-18 14:17:29,704 INFO [main] http.HttpServer: Jetty bound to port 16010 2023-09-18 14:17:29,706 INFO [main] server.Server: jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-8u372-ga~us1-0ubuntu1~18.04-b09 2023-09-18 14:17:29,742 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-09-18 14:17:29,747 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.s.ServletContextHandler@765650e4{logs,/logs,file:///var/log/hbase/,AVAILABLE} 2023-09-18 14:17:29,747 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-09-18 14:17:29,748 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.s.ServletContextHandler@2fc72df3{static,/static,file:///hbase/hbase-2.4.17/hbase-webapps/static/,AVAILABLE} 2023-09-18 14:17:29,858 INFO [main] webapp.StandardDescriptorProcessor: NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-09-18 14:17:29,873 INFO [main] server.session: DefaultSessionIdManager workerName=node0 2023-09-18 14:17:29,873 INFO [main] server.session: No SessionScavenger set, using defaults 2023-09-18 14:17:29,876 INFO [main] server.session: node0 Scavenging every 660000ms 2023-09-18 14:17:29,883 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-09-18 14:17:29,945 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.w.WebAppContext@b7da2c4{master,/,file:///hbase/hbase-2.4.17/hbase-webapps/master/,AVAILABLE}{file:/hbase/hbase-2.4.17/hbase-webapps/master} 2023-09-18 14:17:29,964 INFO [main] server.AbstractConnector: Started ServerConnector@f4deb35{HTTP/1.1, (http/1.1)}{0.0.0.0:16010} 2023-09-18 14:17:29,964 INFO [main] server.Server: Started @10208ms 2023-09-18 14:17:29,976 INFO [main] master.HMaster: hbase.rootdir=hdfs://master:8020/hbase, hbase.cluster.distributed=true 2023-09-18 14:17:30,052 INFO [master/hmaster:16000:becomeActiveMaster] master.HMaster: Adding backup master ZNode /hbase/backup-masters/hmaster,16000,1695046643884 2023-09-18 14:17:30,293 INFO [master/hmaster:16000:becomeActiveMaster] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/hmaster,16000,1695046643884 from backup master directory 2023-09-18 14:17:30,316 INFO [master/hmaster:16000:becomeActiveMaster] master.ActiveMasterManager: Registered as active master=hmaster,16000,1695046643884 2023-09-18 14:17:30,322 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.ChunkCreator: Allocating data MemStoreChunkPool with chunk size 2 MB, max count 5487, initial count 0 2023-09-18 14:17:30,325 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.ChunkCreator: Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 6097, initial count 0 2023-09-18 14:17:31,433 INFO [master/hmaster:16000:becomeActiveMaster] util.FSUtils: Created version file at hdfs://master:8020/hbase with version=8 2023-09-18 14:17:31,767 INFO [master/hmaster:16000:becomeActiveMaster] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-09-18 14:17:32,000 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb] zookeeper.ZooKeeper: Initiating client connection, connectString=hmaster:2181,hregion1:2181,hregion2:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$93/2069699731@3a6016a8 2023-09-18 14:17:32,001 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb] zookeeper.ClientCnxnSocket: jute.maxbuffer value is 4194304 Bytes 2023-09-18 14:17:32,001 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb] zookeeper.ClientCnxn: zookeeper.request.timeout value is 0. feature enabled= 2023-09-18 14:17:32,003 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Opening socket connection to server hregion2/192.168.243.4:2181. Will not attempt to authenticate using SASL (unknown error) 2023-09-18 14:17:32,003 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Socket connection established, initiating session, client: /192.168.243.2:59392, server: hregion2/192.168.243.4:2181 2023-09-18 14:17:32,019 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Session establishment complete on server hregion2/192.168.243.4:2181, sessionid = 0x20050be75b10001, negotiated timeout = 90000 2023-09-18 14:17:32,166 INFO [master/hmaster:16000:becomeActiveMaster] region.MasterRegion: Create or load local region for table 'master:store', {NAME => 'proc', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'} 2023-09-18 14:17:32,171 INFO [master/hmaster:16000:becomeActiveMaster] region.MasterRegionFlusherAndCompactor: Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-09-18 14:17:32,235 INFO [master/hmaster:16000:becomeActiveMaster] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-09-18 14:17:32,389 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {NAME => 'proc', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}, under table dir hdfs://master:8020/hbase/MasterData/data/master/store-tmp 2023-09-18 14:17:32,488 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-18 14:17:32,488 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-18 14:17:32,609 INFO [master/hmaster:16000:becomeActiveMaster] wal.AbstractFSWAL: WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hmaster%2C16000%2C1695046643884, suffix=, logDir=hdfs://master:8020/hbase/MasterData/WALs/hmaster,16000,1695046643884, archiveDir=hdfs://master:8020/hbase/MasterData/oldWALs, maxLogs=10 2023-09-18 14:17:32,868 INFO [master/hmaster:16000:becomeActiveMaster] wal.AbstractFSWAL: New WAL /hbase/MasterData/WALs/hmaster,16000,1695046643884/hmaster%2C16000%2C1695046643884.1695046652627 2023-09-18 14:17:33,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1] regionserver.HStore: Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-09-18 14:17:33,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1] compactions.CompactionConfiguration: size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-09-18 14:17:33,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1] regionserver.HStore: Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=HOT, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-09-18 14:17:33,342 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10442787040, jitterRate=-0.027439668774604797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-09-18 14:17:33,346 INFO [master/hmaster:16000:becomeActiveMaster] region.MasterRegionFlusherAndCompactor: Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-09-18 14:17:33,404 INFO [master/hmaster:16000:becomeActiveMaster] region.RegionProcedureStore: Starting the Region Procedure Store, number threads=16 2023-09-18 14:17:33,404 INFO [master/hmaster:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Starting 16 core workers (bigger of cpus/4 or 16) with max (burst) worker count=160 2023-09-18 14:17:33,409 INFO [master/hmaster:16000:becomeActiveMaster] region.RegionProcedureStore: Starting Region Procedure Store lease recovery... 2023-09-18 14:17:33,412 INFO [master/hmaster:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Recovered RegionProcedureStore lease in 2 msec 2023-09-18 14:17:33,505 INFO [master/hmaster:16000:becomeActiveMaster] procedure2.ProcedureExecutor: Loaded RegionProcedureStore in 92 msec 2023-09-18 14:17:33,505 INFO [master/hmaster:16000:becomeActiveMaster] procedure2.RemoteProcedureDispatcher: Instantiated, coreThreads=128 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-09-18 14:17:33,613 INFO [master/hmaster:16000:becomeActiveMaster] master.RegionServerTracker: Starting RegionServerTracker; 0 have existing ServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-09-18 14:17:33,747 INFO [master/hmaster:16000:becomeActiveMaster] balancer.BaseLoadBalancer: slop=0.001, systemTablesOnMaster=false 2023-09-18 14:17:33,767 INFO [master/hmaster:16000:becomeActiveMaster] balancer.StochasticLoadBalancer: Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-09-18 14:17:33,804 INFO [master/hmaster:16000:becomeActiveMaster] normalizer.RegionNormalizerWorker: Normalizer rate limit set to unlimited 2023-09-18 14:17:33,951 INFO [master/hmaster:16000:becomeActiveMaster] master.HMaster: Active/primary master=hmaster,16000,1695046643884, sessionid=0x10050be756c0001, setting cluster-up flag (Was=false) 2023-09-18 14:17:34,330 WARN [master/hmaster:16000:becomeActiveMaster] snapshot.SnapshotManager: Couldn't delete working snapshot directory: hdfs://master:8020/hbase/.hbase-snapshot/.tmp 2023-09-18 14:17:34,449 INFO [master/hmaster:16000:becomeActiveMaster] coprocessor.CoprocessorHost: System coprocessor org.apache.hadoop.hbase.quotas.MasterQuotasObserver loaded, priority=536870911. 2023-09-18 14:17:34,660 INFO [master/hmaster:16000:becomeActiveMaster] procedure2.TimeoutExecutorThread: ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1695046684660 2023-09-18 14:17:34,667 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.DirScanPool: log_cleaner Cleaner pool size is 1 2023-09-18 14:17:34,670 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-09-18 14:17:34,678 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-09-18 14:17:34,679 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-09-18 14:17:34,680 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-09-18 14:17:34,680 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.LogCleaner: Creating 2 old WALs cleaner threads 2023-09-18 14:17:34,681 INFO [PEWorker-2] procedure.InitMetaProcedure: BOOTSTRAP: creating hbase:meta region 2023-09-18 14:17:34,681 INFO [master/hmaster:16000:becomeActiveMaster] hbase.ChoreService: Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-18 14:17:34,684 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.DirScanPool: hfile_cleaner Cleaner pool size is 10 2023-09-18 14:17:34,687 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-09-18 14:17:34,688 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-09-18 14:17:34,695 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-09-18 14:17:34,697 INFO [master/hmaster:16000:becomeActiveMaster] cleaner.CleanerChore: Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-09-18 14:17:34,701 INFO [PEWorker-2] util.FSTableDescriptors: table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|'}}, {NAME => 'info', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '8192', REPLICATION_SCOPE => '0'}, {NAME => 'rep_barrier', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}, {NAME => 'table', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '8192', REPLICATION_SCOPE => '0'} 2023-09-18 14:17:34,702 INFO [master/hmaster:16000:becomeActiveMaster] hbase.ChoreService: Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-09-18 14:17:34,702 INFO [master/hmaster:16000:becomeActiveMaster] master.HMaster: Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-09-18 14:17:34,708 INFO [master/hmaster:16000:becomeActiveMaster] hbase.ChoreService: Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-09-18 14:17:34,708 INFO [master/hmaster:16000:becomeActiveMaster] hbase.ChoreService: Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-09-18 14:17:34,733 INFO [HBase-Metrics2-1] impl.GlobalMetricRegistriesAdapter: Registering Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.quotas.MasterQuotasObserver Metrics about HBase MasterObservers 2023-09-18 14:17:34,810 INFO [PEWorker-2] util.FSTableDescriptors: Updated hbase:meta table descriptor to hdfs://master:8020/hbase/data/hbase/meta/.tabledesc/.tableinfo.0000000001 2023-09-18 14:17:34,811 INFO [PEWorker-2] regionserver.HRegion: LE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|'}}, {NAME => 'info', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '8192', REPLICATION_SCOPE => '0'}, {NAME => 'rep_barrier', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}, {NAME => 'table', BLOOMFILTER => 'NONE', IN_MEMORY => 'true', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '8192', REPLICATION_SCOPE => '0'}, regionDir=hdfs://master:8020/hbase 2023-09-18 14:17:34,854 WARN [Thread-37] hdfs.DataStreamer: DataStreamer Exception org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /hbase/data/hbase/meta/1588230740/.regioninfo could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1832) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:265) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2586) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:889) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:517) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1540) at org.apache.hadoop.ipc.Client.call(Client.java:1486) at org.apache.hadoop.ipc.Client.call(Client.java:1385) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) at com.sun.proxy.$Proxy19.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:448) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy20.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DataStreamer.locateFollowingBlock(DataStreamer.java:1846) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1645) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:710) 2023-09-18 14:17:34,862 WARN [PEWorker-2] procedure.InitMetaProcedure: Failed to init meta, suspend 1000secs org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /hbase/data/hbase/meta/1588230740/.regioninfo could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1832) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:265) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2586) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:889) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:517) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1540) at org.apache.hadoop.ipc.Client.call(Client.java:1486) at org.apache.hadoop.ipc.Client.call(Client.java:1385) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) at com.sun.proxy.$Proxy19.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:448) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy20.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at org.apache.hadoop.hdfs.DataStreamer.locateFollowingBlock(DataStreamer.java:1846) at org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1645) at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:710) 2023-09-18 14:17:34,865 INFO [PEWorker-2] procedure2.TimeoutExecutorThread: ADDED pid=1, state=WAITING_TIMEOUT:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta; timeout=1000, timestamp=1695046655865 2023-09-18 14:17:35,093 INFO [RpcServer.default.FPBQ.Fifo.handler=29,queue=2,port=16000] master.ServerManager: Registering regionserver=hregion1,16020,1695046641158 2023-09-18 14:17:35,094 INFO [RpcServer.default.FPBQ.Fifo.handler=28,queue=1,port=16000] master.ServerManager: Registering regionserver=hregion2,16020,1695046641063 2023-09-18 14:17:35,250 INFO [RegionServerTracker-0] master.RegionServerTracker: RegionServer ephemeral node created, adding [hregion2,16020,1695046641063] 2023-09-18 14:17:35,251 INFO [RegionServerTracker-0] master.RegionServerTracker: RegionServer ephemeral node created, adding [hregion1,16020,1695046641158] 2023-09-18 14:17:35,922 ERROR [PEWorker-1] procedure2.ProcedureExecutor: CODE-BUG: Uncaught runtime exception for pid=1, state=FAILED:INIT_META_WRITE_FS_LAYOUT, locked=true, exception=org.apache.hadoop.hbase.exceptions.TimeoutIOException via ProcedureExecutor:org.apache.hadoop.hbase.exceptions.TimeoutIOException: Operation timed out after 1.0010 sec; InitMetaProcedure table=hbase:meta java.lang.UnsupportedOperationException at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.rollbackState(InitMetaProcedure.java:132) at org.apache.hadoop.hbase.master.procedure.InitMetaProcedure.rollbackState(InitMetaProcedure.java:50) at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.rollback(StateMachineProcedure.java:212) at org.apache.hadoop.hbase.procedure2.Procedure.doRollback(Procedure.java:938) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1565) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1498) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1349) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1100(ProcedureExecutor.java:73) at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1964) 2023-09-18 14:17:35,931 INFO [PEWorker-1] procedure2.ProcedureExecutor: Rolled back pid=1, state=ROLLEDBACK, exception=org.apache.hadoop.hbase.exceptions.TimeoutIOException via ProcedureExecutor:org.apache.hadoop.hbase.exceptions.TimeoutIOException: Operation timed out after 1.0010 sec; InitMetaProcedure table=hbase:meta exec-time=1.4660 sec 2023-09-18 14:17:35,931 INFO [master/hmaster:16000:becomeActiveMaster] master.HMaster: Wait for region servers to report in: status=null, state=RUNNING, startTime=1695046655931, completionTime=-1 2023-09-18 14:17:35,932 INFO [master/hmaster:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=2; waited=0ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=0ms 2023-09-18 14:17:37,438 INFO [master/hmaster:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=2; waited=1505ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=1505ms 2023-09-18 14:17:38,941 INFO [master/hmaster:16000:becomeActiveMaster] master.ServerManager: Waiting on regionserver count=2; waited=3009ms, expecting min=1 server(s), max=NO_LIMIT server(s), timeout=4500ms, lastChange=3009ms 2023-09-18 14:17:40,445 INFO [master/hmaster:16000:becomeActiveMaster] master.ServerManager: Finished waiting on RegionServer count=2; waited=4513ms, expected min=1 server(s), max=NO_LIMIT server(s), master is running 2023-09-18 14:17:40,452 ERROR [master/hmaster:16000:becomeActiveMaster] master.HMaster: Failed to become active master java.lang.NullPointerException at org.apache.hadoop.hbase.master.HMaster.isRegionOnline(HMaster.java:1229) at org.apache.hadoop.hbase.master.HMaster.waitForMetaOnline(HMaster.java:1218) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:968) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) 2023-09-18 14:17:40,453 ERROR [master/hmaster:16000:becomeActiveMaster] master.HMaster: Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.quotas.MasterQuotasObserver] 2023-09-18 14:17:40,453 ERROR [master/hmaster:16000:becomeActiveMaster] master.HMaster: ***** ABORTING master hmaster,16000,1695046643884: Unhandled exception. Starting shutdown. ***** java.lang.NullPointerException at org.apache.hadoop.hbase.master.HMaster.isRegionOnline(HMaster.java:1229) at org.apache.hadoop.hbase.master.HMaster.waitForMetaOnline(HMaster.java:1218) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:968) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) 2023-09-18 14:17:40,457 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegionServer: ***** STOPPING region server 'hmaster,16000,1695046643884' ***** 2023-09-18 14:17:40,457 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegionServer: STOPPED: Stopped by master/hmaster:16000:becomeActiveMaster 2023-09-18 14:17:40,459 INFO [master/hmaster:16000] regionserver.HRegionServer: Stopping infoServer 2023-09-18 14:17:40,484 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.w.WebAppContext@b7da2c4{master,/,null,STOPPED}{file:/hbase/hbase-2.4.17/hbase-webapps/master} 2023-09-18 14:17:40,492 INFO [master/hmaster:16000] server.AbstractConnector: Stopped ServerConnector@f4deb35{HTTP/1.1, (http/1.1)}{0.0.0.0:16010} 2023-09-18 14:17:40,493 INFO [master/hmaster:16000] server.session: node0 Stopped scavenging 2023-09-18 14:17:40,495 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2fc72df3{static,/static,file:///hbase/hbase-2.4.17/hbase-webapps/static/,STOPPED} 2023-09-18 14:17:40,496 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.s.ServletContextHandler@765650e4{logs,/logs,file:///var/log/hbase/,STOPPED} 2023-09-18 14:17:40,501 INFO [master/hmaster:16000] regionserver.HRegionServer: aborting server hmaster,16000,1695046643884 2023-09-18 14:17:40,501 INFO [master/hmaster:16000] regionserver.HRegionServer: stopping server hmaster,16000,1695046643884; all regions closed. 2023-09-18 14:17:40,502 WARN [OldWALsCleaner-1] cleaner.LogCleaner: Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-18 14:17:40,502 WARN [OldWALsCleaner-0] cleaner.LogCleaner: Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-09-18 14:17:40,504 INFO [master/hmaster:16000] hbase.ChoreService: Chore service for: master/hmaster:16000 had [] on shutdown 2023-09-18 14:17:40,506 INFO [master/hmaster:16000] procedure2.RemoteProcedureDispatcher: Stopping procedure remote dispatcher 2023-09-18 14:17:40,506 INFO [master/hmaster:16000] procedure2.ProcedureExecutor: Stopping 2023-09-18 14:17:40,512 INFO [master/hmaster:16000] region.RegionProcedureStore: Stopping the Region Procedure Store, isAbort=true 2023-09-18 14:17:40,514 WARN [master/hmaster:16000] master.ActiveMasterManager: Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-09-18 14:17:40,514 INFO [master/hmaster:16000] assignment.AssignmentManager: Stopping assignment manager 2023-09-18 14:17:40,515 INFO [master/hmaster:16000] region.MasterRegion: Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2023-09-18 14:17:40,549 INFO [master/hmaster:16000] regionserver.HRegion: Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-18 14:17:40,553 ERROR [master/hmaster:16000] regionserver.HRegion: Memstore data size is 2937 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-18 14:17:40,554 INFO [master/hmaster:16000] regionserver.HRegion: Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-09-18 14:17:40,554 INFO [master/hmaster:16000] flush.MasterFlushTableProcedureManager: stop: server shutting down. 2023-09-18 14:17:40,554 INFO [master:store-WAL-Roller] wal.AbstractWALRoller: LogRoller exiting. 2023-09-18 14:17:40,556 INFO [master/hmaster:16000] ipc.NettyRpcServer: Stopping server on /192.168.243.2:16000 2023-09-18 14:17:40,624 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb] zookeeper.ZooKeeper: Session: 0x20050be75b10001 closed 2023-09-18 14:17:40,624 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x057b6ceb-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x20050be75b10001 2023-09-18 14:17:40,722 INFO [master/hmaster:16000] zookeeper.ZooKeeper: Session: 0x10050be756c0001 closed 2023-09-18 14:17:40,723 INFO [master/hmaster:16000] regionserver.HRegionServer: Exiting; stopping=hmaster,16000,1695046643884; zookeeper connection closed. 2023-09-18 14:17:40,723 ERROR [main-EventThread] zookeeper.ClientCnxn: Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@533d7002 rejected from java.util.concurrent.ThreadPoolExecutor@4be1b413[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at java.util.concurrent.Executors$DelegatedExecutorService.submit(Executors.java:678) at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:602) at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:535) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) 2023-09-18 14:17:40,723 ERROR [main] master.HMasterCommandLine: Master exiting java.lang.RuntimeException: HMaster Aborted at org.apache.hadoop.hbase.master.HMasterCommandLine.startMaster(HMasterCommandLine.java:254) at org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:145) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:140) at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:2963) 2023-09-18 14:17:40,724 ERROR [main-EventThread] zookeeper.ClientCnxn: Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@2ee73ce5 rejected from java.util.concurrent.ThreadPoolExecutor@4be1b413[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 15] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at java.util.concurrent.Executors$DelegatedExecutorService.submit(Executors.java:678) at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:602) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:535) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) 2023-09-18 14:17:40,725 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x10050be756c0001