Tue Oct 17 21:02:48 UTC 2023 Starting master on a82083cf5d18 core file size (blocks, -c) unlimited data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 768033 max locked memory (kbytes, -l) unlimited max memory size (kbytes, -m) unlimited open files (-n) 100000 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 32768 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited 2023-10-17 21:02:54,897 INFO [main] master.HMaster: STARTING service HMaster 2023-10-17 21:02:54,903 INFO [main] util.VersionInfo: HBase 2.4.17 2023-10-17 21:02:54,903 INFO [main] util.VersionInfo: Source code repository git://e0b8ecc3178f/home/taklwu/hbase-rm/output/hbase revision=7fd096f39b4284da9a71da3ce67c48d259ffa79a 2023-10-17 21:02:54,903 INFO [main] util.VersionInfo: Compiled by taklwu on Fri Mar 31 18:10:45 UTC 2023 2023-10-17 21:02:54,903 INFO [main] util.VersionInfo: From source with checksum 0e34884c8e1d6e46ba560b6a824d684ba4dd42b8f34a4318e57510627a88e3e41e34bdb8f06da56a46d8065f331aa349ad58755cf6db5d75a5742f4174ccf81d 2023-10-17 21:02:56,003 INFO [main] util.ServerCommandLine: hbase.tmp.dir: ./tmp 2023-10-17 21:02:56,004 INFO [main] util.ServerCommandLine: hbase.rootdir: hdfs://master:8020/hbase 2023-10-17 21:02:56,005 INFO [main] util.ServerCommandLine: hbase.cluster.distributed: true 2023-10-17 21:02:56,005 INFO [main] util.ServerCommandLine: hbase.zookeeper.quorum: hmaster,hregion1,hregion2 2023-10-17 21:02:56,016 INFO [main] util.ServerCommandLine: env:PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 2023-10-17 21:02:56,017 INFO [main] util.ServerCommandLine: env:HBASE_LOGFILE=hbase--master-a82083cf5d18.log 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:SUPERVISOR_ENABLED=1 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:IS_HMASTER=true 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:PYTHON=python2 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:HBASE_CONF_DIR=/etc/hbase-2.4.17 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:HBASE_SHELL_DAEMON_PORT=36000 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:CUR_STATUS=ORI 2023-10-17 21:02:56,024 INFO [main] util.ServerCommandLine: env:HBASE_SHELL_HOST=192.168.132.2 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:JAVA_TOOL_OPTIONS=-javaagent:/org.jacoco.agent.rt.jar=append=false,includes=org.apache.hadoop.hbase.*,excludes=org.jruby.*,output=dfe,address=192.168.132.1,port=39113,sessionid=hbase-sCr4V3nf_original-0 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_REST_OPTS= 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:JVM_PID=673 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:PWD=/ 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HADOOP_IP=192.168.132.102 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_SEEDS=192.168.132.2, 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_ROOT_LOGGER=INFO,RFA 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_ENV_INIT=true 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_IDENT_STRING= 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_ZNODE_FILE=/tmp/hbase--master.znode 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_LOG_DIR=/var/log/hbase 2023-10-17 21:02:56,025 INFO [main] util.ServerCommandLine: env:HBASE_LOG_PREFIX=hbase--master-a82083cf5d18 2023-10-17 21:02:56,027 INFO [main] util.ServerCommandLine: ose-jwt-4.41.1.jar:/hbase/hbase-2.4.17/lib/objenesis-2.6.jar:/hbase/hbase-2.4.17/lib/okhttp-2.7.5.jar:/hbase/hbase-2.4.17/lib/okio-1.6.0.jar:/hbase/hbase-2.4.17/lib/paranamer-2.3.jar:/hbase/hbase-2.4.17/lib/protobuf-java-2.5.0.jar:/hbase/hbase-2.4.17/lib/snappy-java-1.0.5.jar:/hbase/hbase-2.4.17/lib/spymemcached-2.12.2.jar:/hbase/hbase-2.4.17/lib/stax2-api-3.1.4.jar:/hbase/hbase-2.4.17/lib/woodstox-core-5.0.3.jar:/hbase/hbase-2.4.17/lib/xmlenc-0.52.jar:/hbase/hbase-2.4.17/lib/zookeeper-3.5.7.jar:/hbase/hbase-2.4.17/lib/zookeeper-jute-3.5.7.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/audience-annotations-0.13.0.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/commons-logging-1.2.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/reload4j-1.2.22.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-api-1.7.33.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:SUPERVISOR_SERVER_URL=unix:///var/run/supervisor.sock 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:HBASE_CLUSTER_NAME=dev_cluster 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:HBASE_LOGGING_LEVEL=DEBUG 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:HBASE_AUTOSTART_FILE=/tmp/hbase--master.autostart 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:SED=sed 2023-10-17 21:02:56,036 INFO [main] util.ServerCommandLine: env:HOSTNAME=a82083cf5d18 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:SUPERVISOR_GROUP_NAME=upfuzz_hbase 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:SUPERVISOR_PROCESS_NAME=hbase 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:GREP=grep 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_NICENESS=0 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_SECURITY_LOGGER=INFO,RFAS 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_OPTS=-XX:+UseConcMarkSweepGC -Djava.util.logging.config.class=org.apache.hadoop.hbase.logging.JulToSlf4jInitializer -Dhbase.log.dir=/var/log/hbase -Dhbase.log.file=hbase--master-a82083cf5d18.log -Dhbase.home.dir=/hbase/hbase-2.4.17 -Dhbase.id.str= -Dhbase.root.logger=INFO,RFA -Dhbase.security.logger=INFO,RFAS 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_HOME=/hbase/hbase-2.4.17 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_THRIFT_OPTS= 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:HBASE_CONF=/etc/hbase-2.4.17 2023-10-17 21:02:56,037 INFO [main] util.ServerCommandLine: env:SHLVL=5 2023-10-17 21:02:56,038 INFO [main] util.ServerCommandLine: env:HOME=/root 2023-10-17 21:02:56,038 INFO [main] util.ServerCommandLine: env:MALLOC_ARENA_MAX=4 2023-10-17 21:02:56,040 INFO [main] util.ServerCommandLine: vmName=OpenJDK 64-Bit Server VM, vmVendor=Private Build, vmVersion=25.362-b09 2023-10-17 21:02:56,041 INFO [main] util.ServerCommandLine: vmInputArguments=[-javaagent:/org.jacoco.agent.rt.jar=append=false,includes=org.apache.hadoop.hbase.*,excludes=org.jruby.*,output=dfe,address=192.168.132.1,port=39113,sessionid=hbase-sCr4V3nf_original-0, -Dproc_master, -XX:OnOutOfMemoryError=kill -9 %p, -XX:+UseConcMarkSweepGC, -Djava.util.logging.config.class=org.apache.hadoop.hbase.logging.JulToSlf4jInitializer, -Dhbase.log.dir=/var/log/hbase, -Dhbase.log.file=hbase--master-a82083cf5d18.log, -Dhbase.home.dir=/hbase/hbase-2.4.17, -Dhbase.id.str=, -Dhbase.root.logger=INFO,RFA, -Dhbase.security.logger=INFO,RFAS] 2023-10-17 21:02:59,022 INFO [main] metrics.MetricRegistries: Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-17 21:02:59,464 WARN [main] util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-17 21:03:06,167 INFO [main] regionserver.RSRpcServices: master/hmaster:16000 server-side Connection retries=45 2023-10-17 21:03:06,606 INFO [main] ipc.RpcExecutor: Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=3, maxQueueLength=300, handlerCount=30 2023-10-17 21:03:06,621 INFO [main] ipc.RpcExecutor: Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=300, handlerCount=20 2023-10-17 21:03:06,622 INFO [main] ipc.RWQueueRpcExecutor: priority.RWQ.Fifo writeQueues=1 writeHandlers=2 readQueues=1 readHandlers=18 scanQueues=0 scanHandlers=0 2023-10-17 21:03:06,622 INFO [main] ipc.RpcExecutor: Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=300, handlerCount=3 2023-10-17 21:03:06,628 INFO [main] ipc.RpcExecutor: Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=300, handlerCount=1 2023-10-17 21:03:09,515 INFO [main] ipc.RpcServerFactory: Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService 2023-10-17 21:03:09,957 INFO [main] Configuration.deprecation: hbase.ipc.server.reservoir.initial.buffer.size is deprecated. Instead, use hbase.server.allocator.buffer.size 2023-10-17 21:03:09,957 INFO [main] Configuration.deprecation: hbase.ipc.server.reservoir.initial.max is deprecated. Instead, use hbase.server.allocator.max.buffer.count 2023-10-17 21:03:12,050 INFO [main] ipc.NettyRpcServer: Bind to /192.168.132.2:16000 2023-10-17 21:03:17,941 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-17 21:03:17,985 INFO [main] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-17 21:03:19,078 INFO [main] zookeeper.RecoverableZooKeeper: Process identifier=master:16000 connecting to ZooKeeper ensemble=hmaster:2181,hregion1:2181,hregion2:2181 2023-10-17 21:03:19,164 INFO [main] zookeeper.ZooKeeper: Client environment:zookeeper.version=3.5.7-f0fdd52973d373ffd9c86b81d99842dc2c7f660e, built on 02/10/2020 11:30 GMT 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:host.name=hmaster 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.version=1.8.0_362 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.vendor=Private Build 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: /hbase-2.4.17/lib/objenesis-2.6.jar:/hbase/hbase-2.4.17/lib/okhttp-2.7.5.jar:/hbase/hbase-2.4.17/lib/okio-1.6.0.jar:/hbase/hbase-2.4.17/lib/paranamer-2.3.jar:/hbase/hbase-2.4.17/lib/protobuf-java-2.5.0.jar:/hbase/hbase-2.4.17/lib/snappy-java-1.0.5.jar:/hbase/hbase-2.4.17/lib/spymemcached-2.12.2.jar:/hbase/hbase-2.4.17/lib/stax2-api-3.1.4.jar:/hbase/hbase-2.4.17/lib/woodstox-core-5.0.3.jar:/hbase/hbase-2.4.17/lib/xmlenc-0.52.jar:/hbase/hbase-2.4.17/lib/zookeeper-3.5.7.jar:/hbase/hbase-2.4.17/lib/zookeeper-jute-3.5.7.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/audience-annotations-0.13.0.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/commons-logging-1.2.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/htrace-core4-4.2.0-incubating.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/reload4j-1.2.22.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-api-1.7.33.jar:/hbase/hbase-2.4.17/lib/client-facing-thirdparty/slf4j-reload4j-1.7.33.jar:/org.jacoco.agent.rt.jar 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:java.compiler= 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.name=Linux 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.arch=amd64 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.version=6.2.0-26-generic 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:user.name=root 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:user.home=/root 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:user.dir=/ 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.free=1771MB 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.max=30485MB 2023-10-17 21:03:19,165 INFO [main] zookeeper.ZooKeeper: Client environment:os.memory.total=1979MB 2023-10-17 21:03:19,190 INFO [main] zookeeper.ZooKeeper: Initiating client connection, connectString=hmaster:2181,hregion1:2181,hregion2:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.PendingWatcher@216906a0 2023-10-17 21:03:19,213 INFO [main] common.X509Util: Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation 2023-10-17 21:03:19,282 INFO [main] zookeeper.ClientCnxnSocket: jute.maxbuffer value is 4194304 Bytes 2023-10-17 21:03:19,343 INFO [main] zookeeper.ClientCnxn: zookeeper.request.timeout value is 0. feature enabled= 2023-10-17 21:03:19,945 INFO [main-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Opening socket connection to server hregion2/192.168.132.4:2181. Will not attempt to authenticate using SASL (unknown error) 2023-10-17 21:03:20,004 INFO [main-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Socket connection established, initiating session, client: /192.168.132.2:57226, server: hregion2/192.168.132.4:2181 2023-10-17 21:03:20,328 INFO [main-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Session establishment complete on server hregion2/192.168.132.4:2181, sessionid = 0x20003a8cb2f0000, negotiated timeout = 90000 2023-10-17 21:03:21,991 INFO [main] util.log: Logging initialized @31598ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-17 21:03:23,090 INFO [main] http.HttpServer: Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-17 21:03:23,092 INFO [main] http.HttpServer: Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-17 21:03:23,106 INFO [main] http.HttpServer: Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-17 21:03:23,140 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-17 21:03:23,154 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-17 21:03:23,154 INFO [main] http.HttpServer: Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-17 21:03:23,206 INFO [main] http.HttpServer: ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-17 21:03:23,499 INFO [main] http.HttpServer: Jetty bound to port 16010 2023-10-17 21:03:23,501 INFO [main] server.Server: jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-8u372-ga~us1-0ubuntu1~18.04-b09 2023-10-17 21:03:23,872 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-10-17 21:03:23,926 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.s.ServletContextHandler@26753687{logs,/logs,file:///var/log/hbase/,AVAILABLE} 2023-10-17 21:03:23,927 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-10-17 21:03:23,927 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.s.ServletContextHandler@358caf3a{static,/static,file:///hbase/hbase-2.4.17/hbase-webapps/static/,AVAILABLE} 2023-10-17 21:03:24,355 INFO [main] webapp.StandardDescriptorProcessor: NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-17 21:03:24,418 INFO [main] server.session: DefaultSessionIdManager workerName=node0 2023-10-17 21:03:24,418 INFO [main] server.session: No SessionScavenger set, using defaults 2023-10-17 21:03:24,435 INFO [main] server.session: node0 Scavenging every 600000ms 2023-10-17 21:03:24,449 INFO [main] http.SecurityHeadersFilter: Added security headers filter 2023-10-17 21:03:24,649 INFO [main] handler.ContextHandler: Started o.a.h.t.o.e.j.w.WebAppContext@b14a2c{master,/,file:///hbase/hbase-2.4.17/hbase-webapps/master/,AVAILABLE}{file:/hbase/hbase-2.4.17/hbase-webapps/master} 2023-10-17 21:03:24,714 INFO [main] server.AbstractConnector: Started ServerConnector@53d782b0{HTTP/1.1, (http/1.1)}{0.0.0.0:16010} 2023-10-17 21:03:24,715 INFO [main] server.Server: Started @34322ms 2023-10-17 21:03:24,727 INFO [main] master.HMaster: hbase.rootdir=hdfs://master:8020/hbase, hbase.cluster.distributed=true 2023-10-17 21:03:25,344 INFO [master/hmaster:16000:becomeActiveMaster] master.HMaster: Adding backup master ZNode /hbase/backup-masters/hmaster,16000,1697576576301 2023-10-17 21:03:26,981 INFO [master/hmaster:16000:becomeActiveMaster] master.ActiveMasterManager: Deleting ZNode for /hbase/backup-masters/hmaster,16000,1697576576301 from backup master directory 2023-10-17 21:03:27,109 INFO [master/hmaster:16000:becomeActiveMaster] master.ActiveMasterManager: Registered as active master=hmaster,16000,1697576576301 2023-10-17 21:03:27,121 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.ChunkCreator: Allocating data MemStoreChunkPool with chunk size 2 MB, max count 5487, initial count 0 2023-10-17 21:03:27,127 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.ChunkCreator: Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 6097, initial count 0 2023-10-17 21:03:29,945 INFO [master/hmaster:16000:becomeActiveMaster] util.FSUtils: Created version file at hdfs://master:8020/hbase with version=8 2023-10-17 21:03:31,244 INFO [master/hmaster:16000:becomeActiveMaster] fs.HFileSystem: Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-17 21:03:33,565 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033] zookeeper.ZooKeeper: Initiating client connection, connectString=hmaster:2181,hregion1:2181,hregion2:2181 sessionTimeout=90000 watcher=org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$83/653105918@67d090c5 2023-10-17 21:03:33,629 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033] zookeeper.ClientCnxnSocket: jute.maxbuffer value is 4194304 Bytes 2023-10-17 21:03:33,630 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033] zookeeper.ClientCnxn: zookeeper.request.timeout value is 0. feature enabled= 2023-10-17 21:03:33,664 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Opening socket connection to server hregion2/192.168.132.4:2181. Will not attempt to authenticate using SASL (unknown error) 2023-10-17 21:03:33,665 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Socket connection established, initiating session, client: /192.168.132.2:48544, server: hregion2/192.168.132.4:2181 2023-10-17 21:03:33,716 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033-SendThread(hregion2:2181)] zookeeper.ClientCnxn: Session establishment complete on server hregion2/192.168.132.4:2181, sessionid = 0x20003a8cb2f0001, negotiated timeout = 90000 2023-10-17 21:03:34,626 INFO [master/hmaster:16000:becomeActiveMaster] region.MasterRegion: Create or load local region for table 'master:store', {NAME => 'proc', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'} 2023-10-17 21:03:34,645 INFO [master/hmaster:16000:becomeActiveMaster] region.MasterRegionFlusherAndCompactor: Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-17 21:03:34,817 INFO [master/hmaster:16000:becomeActiveMaster] wal.WALFactory: Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-17 21:03:35,192 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {NAME => 'proc', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', COMPRESSION => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', BLOCKCACHE => 'true', BLOCKSIZE => '65536', REPLICATION_SCOPE => '0'}, under table dir hdfs://master:8020/hbase/MasterData/data/master/store-tmp 2023-10-17 21:03:35,500 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-17 21:03:35,500 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegion: Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-17 21:03:35,673 INFO [master/hmaster:16000:becomeActiveMaster] wal.AbstractFSWAL: WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hmaster%2C16000%2C1697576576301, suffix=, logDir=hdfs://master:8020/hbase/MasterData/WALs/hmaster,16000,1697576576301, archiveDir=hdfs://master:8020/hbase/MasterData/oldWALs, maxLogs=10 2023-10-17 21:03:35,857 WARN [master/hmaster:16000:becomeActiveMaster] asyncfs.FanOutOneBlockAsyncDFSOutputHelper: create fan-out dfs output /hbase/MasterData/WALs/hmaster,16000,1697576576301/hmaster%2C16000%2C1697576576301.1697576615700 failed, retry = 0 org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /hbase/MasterData/WALs/hmaster,16000,1697576576301/hmaster%2C16000%2C1697576576301.1697576615700 could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1832) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:265) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2586) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:889) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:517) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1540) at org.apache.hadoop.ipc.Client.call(Client.java:1486) at org.apache.hadoop.ipc.Client.call(Client.java:1385) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) at com.sun.proxy.$Proxy19.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:448) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy20.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:541) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$400(FanOutOneBlockAsyncDFSOutputHelper.java:112) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:615) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:610) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:623) at org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:53) at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:190) at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:160) at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:116) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:719) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:128) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:881) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:574) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:515) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:160) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:62) at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:295) at org.apache.hadoop.hbase.master.region.MasterRegion.createWAL(MasterRegion.java:200) at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:220) at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:348) at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:104) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:855) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) 2023-10-17 21:03:35,868 WARN [master/hmaster:16000:becomeActiveMaster] wal.AbstractProtobufLogWriter: Init output failed, path=hdfs://master:8020/hbase/MasterData/WALs/hmaster,16000,1697576576301/hmaster%2C16000%2C1697576576301.1697576615700 java.io.IOException: File /hbase/MasterData/WALs/hmaster,16000,1697576576301/hmaster%2C16000%2C1697576576301.1697576615700 could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1832) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:265) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2586) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:889) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:517) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:568) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$400(FanOutOneBlockAsyncDFSOutputHelper.java:112) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:615) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$8.doCall(FanOutOneBlockAsyncDFSOutputHelper.java:610) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:623) at org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutputHelper.createOutput(AsyncFSOutputHelper.java:53) at org.apache.hadoop.hbase.regionserver.wal.AsyncProtobufLogWriter.initOutput(AsyncProtobufLogWriter.java:190) at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:160) at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.createAsyncWriter(AsyncFSWALProvider.java:116) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:719) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.createWriterInstance(AsyncFSWAL.java:128) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:881) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:574) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.init(AbstractFSWAL.java:515) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:160) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:62) at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:295) at org.apache.hadoop.hbase.master.region.MasterRegion.createWAL(MasterRegion.java:200) at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:220) at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:348) at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:104) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:855) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /hbase/MasterData/WALs/hmaster,16000,1697576576301/hmaster%2C16000%2C1697576576301.1697576615700 could only be replicated to 0 nodes instead of minReplication (=1). There are 1 datanode(s) running and no node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:1832) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:265) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2586) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:889) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:517) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1540) at org.apache.hadoop.ipc.Client.call(Client.java:1486) at org.apache.hadoop.ipc.Client.call(Client.java:1385) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:232) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) at com.sun.proxy.$Proxy19.addBlock(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:448) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy20.addBlock(Unknown Source) at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:361) at com.sun.proxy.$Proxy21.addBlock(Unknown Source) at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.createOutput(FanOutOneBlockAsyncDFSOutputHelper.java:541) ... 25 more 2023-10-17 21:03:35,887 ERROR [master/hmaster:16000:becomeActiveMaster] master.HMaster: Failed to become active master org.apache.hadoop.hbase.FailedCloseWALAfterInitializedErrorException: Failed close after init wal failed. at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:167) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:62) at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:295) at org.apache.hadoop.hbase.master.region.MasterRegion.createWAL(MasterRegion.java:200) at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:220) at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:348) at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:104) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:855) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) Caused by: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.shutdown(AbstractFSWAL.java:979) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.close(AbstractFSWAL.java:1006) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:165) ... 10 more Caused by: java.lang.NullPointerException at java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1011) at java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1006) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.closeWriter(AsyncFSWAL.java:743) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doShutdown(AsyncFSWAL.java:800) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$2.call(AbstractFSWAL.java:951) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$2.call(AbstractFSWAL.java:946) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more 2023-10-17 21:03:35,892 ERROR [master/hmaster:16000:becomeActiveMaster] master.HMaster: ***** ABORTING master hmaster,16000,1697576576301: Unhandled exception. Starting shutdown. ***** org.apache.hadoop.hbase.FailedCloseWALAfterInitializedErrorException: Failed close after init wal failed. at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:167) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:62) at org.apache.hadoop.hbase.wal.WALFactory.getWAL(WALFactory.java:295) at org.apache.hadoop.hbase.master.region.MasterRegion.createWAL(MasterRegion.java:200) at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:220) at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:348) at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:104) at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:855) at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2193) at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:528) at java.lang.Thread.run(Thread.java:750) Caused by: java.io.IOException: java.lang.NullPointerException at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.shutdown(AbstractFSWAL.java:979) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.close(AbstractFSWAL.java:1006) at org.apache.hadoop.hbase.wal.AbstractFSWALProvider.getWAL(AbstractFSWALProvider.java:165) ... 10 more Caused by: java.lang.NullPointerException at java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1011) at java.util.concurrent.ConcurrentHashMap.put(ConcurrentHashMap.java:1006) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.closeWriter(AsyncFSWAL.java:743) at org.apache.hadoop.hbase.regionserver.wal.AsyncFSWAL.doShutdown(AsyncFSWAL.java:800) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$2.call(AbstractFSWAL.java:951) at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$2.call(AbstractFSWAL.java:946) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more 2023-10-17 21:03:35,894 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegionServer: ***** STOPPING region server 'hmaster,16000,1697576576301' ***** 2023-10-17 21:03:35,894 INFO [master/hmaster:16000:becomeActiveMaster] regionserver.HRegionServer: STOPPED: Stopped by master/hmaster:16000:becomeActiveMaster 2023-10-17 21:03:36,074 INFO [master/hmaster:16000] ipc.NettyRpcServer: Stopping server on /192.168.132.2:16000 2023-10-17 21:03:36,108 INFO [master/hmaster:16000] regionserver.HRegionServer: Stopping infoServer 2023-10-17 21:03:36,449 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.w.WebAppContext@b14a2c{master,/,null,STOPPED}{file:/hbase/hbase-2.4.17/hbase-webapps/master} 2023-10-17 21:03:36,480 INFO [master/hmaster:16000] server.AbstractConnector: Stopped ServerConnector@53d782b0{HTTP/1.1, (http/1.1)}{0.0.0.0:16010} 2023-10-17 21:03:36,491 INFO [master/hmaster:16000] server.session: node0 Stopped scavenging 2023-10-17 21:03:36,509 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.s.ServletContextHandler@358caf3a{static,/static,file:///hbase/hbase-2.4.17/hbase-webapps/static/,STOPPED} 2023-10-17 21:03:36,509 INFO [master/hmaster:16000] handler.ContextHandler: Stopped o.a.h.t.o.e.j.s.ServletContextHandler@26753687{logs,/logs,file:///var/log/hbase/,STOPPED} 2023-10-17 21:03:36,531 INFO [master/hmaster:16000] regionserver.HRegionServer: aborting server hmaster,16000,1697576576301 2023-10-17 21:03:36,533 INFO [master/hmaster:16000] regionserver.HRegionServer: stopping server hmaster,16000,1697576576301; all regions closed. 2023-10-17 21:03:36,533 INFO [master/hmaster:16000] hbase.ChoreService: Chore service for: master/hmaster:16000 had [] on shutdown 2023-10-17 21:03:36,568 WARN [master/hmaster:16000] master.ActiveMasterManager: Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-17 21:03:36,660 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033] zookeeper.ZooKeeper: Session: 0x20003a8cb2f0001 closed 2023-10-17 21:03:36,662 INFO [ReadOnlyZKClient-hmaster:2181,hregion1:2181,hregion2:2181@0x2b98f033-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x20003a8cb2f0001 2023-10-17 21:03:36,805 INFO [master/hmaster:16000] zookeeper.ZooKeeper: Session: 0x20003a8cb2f0000 closed 2023-10-17 21:03:36,805 INFO [master/hmaster:16000] regionserver.HRegionServer: Exiting; stopping=hmaster,16000,1697576576301; zookeeper connection closed. 2023-10-17 21:03:36,805 ERROR [main-EventThread] zookeeper.ClientCnxn: Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@82e96f rejected from java.util.concurrent.ThreadPoolExecutor@2aa6497d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 9] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at java.util.concurrent.Executors$DelegatedExecutorService.submit(Executors.java:678) at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:602) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:535) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) 2023-10-17 21:03:36,805 ERROR [main-EventThread] zookeeper.ClientCnxn: Error while calling watcher java.util.concurrent.RejectedExecutionException: Task java.util.concurrent.FutureTask@3e8933fb rejected from java.util.concurrent.ThreadPoolExecutor@2aa6497d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 9] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2063) at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:830) at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1379) at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112) at java.util.concurrent.Executors$DelegatedExecutorService.submit(Executors.java:678) at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:602) at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:535) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:510) 2023-10-17 21:03:36,806 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down for session: 0x20003a8cb2f0000 2023-10-17 21:03:36,805 ERROR [main] master.HMasterCommandLine: Master exiting java.lang.RuntimeException: HMaster Aborted at org.apache.hadoop.hbase.master.HMasterCommandLine.startMaster(HMasterCommandLine.java:254) at org.apache.hadoop.hbase.master.HMasterCommandLine.run(HMasterCommandLine.java:145) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:140) at org.apache.hadoop.hbase.master.HMaster.main(HMaster.java:2963)